{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "language": "python",
   "display_name": "Python 3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import h5py\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import torch\n",
    "from sklearn.metrics import roc_curve, precision_recall_curve, f1_score, roc_auc_score, average_precision_score\n",
    "from torch import sigmoid\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "from rt_bene.blink_estimation_models_pytorch import BlinkEstimationModelResnet18, BlinkEstimationModelResnet50, \\\n",
    "    BlinkEstimationModelVGG19, BlinkEstimationModelDenseNet121, BlinkEstimationModelVGG16\n",
    "from rt_bene_model_training.pytorch.rtbene_dataset import RTBENEH5Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = {\n",
    "    \"resnet18\": BlinkEstimationModelResnet18,\n",
    "    \"resnet50\": BlinkEstimationModelResnet50,\n",
    "    \"vgg16\": BlinkEstimationModelVGG16,\n",
    "    \"vgg19\": BlinkEstimationModelVGG19,\n",
    "    \"densenet121\": BlinkEstimationModelDenseNet121\n",
    "}\n",
    "\n",
    "modes = [\"ensemble\", \"single\"]\n",
    "\n",
    "hdf5_file = os.path.abspath(os.path.expanduser(\"~/datasets/rtbene_dataset.hdf5\"))\n",
    "model_base = \"vgg16\"\n",
    "model_net_dir = os.path.abspath(\"../rt_gene/rt_gbene_model_training/pytorch/model_nets\")\n",
    "mode = \"ensemble\"\n",
    "model_files = glob.glob(os.path.join(model_net_dir, \"*.model\"))\n",
    "\n",
    "assert model_base in models.keys(), \"_model_base requested is not supported\"\n",
    "assert mode in modes, \"prediction mode must be either 'single' or 'ensemble'\"\n",
    "assert len(model_files) > 0, \"No .model files found in directory\"\n",
    "\n",
    "valid_subjects = [0, 11, 15, 16]\n",
    "# create a master list of predictions and labels\n",
    "model_inferences = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "if mode == \"single\":\n",
    "    for model_file in tqdm(model_files, desc=\"Models..\", position=1):\n",
    "        labels = []\n",
    "        predictions = []\n",
    "        model_nn = models.get(model_base)()\n",
    "        model_nn.load_state_dict(torch.load(model_file))\n",
    "        model_nn.cuda()\n",
    "        model_nn.eval()\n",
    "        for subject in tqdm(valid_subjects, desc=\"Subjects...\", position=2):\n",
    "            data_validate = RTBENEH5Dataset(h5_file=h5py.File(hdf5_file, mode=\"r\"), subject_list=[subject], loader_desc=subject)\n",
    "            for index in tqdm(range(0, len(data_validate)), desc=\"Images...\", position=3):\n",
    "                _left, _right, _label = data_validate[index]\n",
    "                _predicted_blink = sigmoid(model_nn(_left.unsqueeze(0).float().cuda(), _right.unsqueeze(0).float().cuda()))\n",
    "                labels.extend(_label)\n",
    "                predictions.append(float(_predicted_blink.detach().cpu()))\n",
    "\n",
    "        model_inferences[model_file] = {\"labels\": labels, \"predictions\": predictions}"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "if mode == \"ensemble\":\n",
    "    model_ensemble = []\n",
    "    for model_file in tqdm(model_files, desc=\"Loading Models..\", position=1):\n",
    "        model_nn = models.get(model_base)()\n",
    "        model_nn.load_state_dict(torch.load(model_file))\n",
    "        model_nn.cuda()\n",
    "        model_nn.eval()\n",
    "        model_ensemble.append(model_nn)\n",
    "\n",
    "    labels = []\n",
    "    predictions = []\n",
    "    for subject in tqdm(valid_subjects, desc=\"Subjects...\", position=2):\n",
    "            data_validate = RTBENEH5Dataset(h5_file=h5py.File(hdf5_file, mode=\"r\"), subject_list=[subject], loader_desc=subject)\n",
    "            for index in tqdm(range(0, len(data_validate)), desc=\"Images...\", position=3):\n",
    "                _left, _right, _label = data_validate[index]\n",
    "                _predicted_blinks = [sigmoid(m(_left.unsqueeze(0).float().cuda(), _right.unsqueeze(0).float().cuda())) for m in model_ensemble]\n",
    "                _predicted_blinks = torch.stack(_predicted_blinks, dim=1)\n",
    "                _predicted_blink = torch.mean(_predicted_blinks, dim=1).detach().cpu()\n",
    "                labels.extend(_label)\n",
    "                predictions.append(float(_predicted_blink))\n",
    "\n",
    "    model_inferences['ensemble'] = {\"labels\": labels, \"predictions\": predictions}"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimal Threshold from ROC 0.75, optimal threshold from PRC 0.99\n"
     ]
    },
    {
     "data": {
      "text/plain": "<Figure size 1080x432 with 2 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3gAAAGDCAYAAAB5pLK9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABUqUlEQVR4nO3dd5gdddXA8e9JpSUUgxASehWkR5qgKL2DdFCkiaAItlfBBnbBLqAQqpQAUqSqSAdpGpDeCSWRElIggRRSzvvHzIbLstndhL07u3u/n+fZZ/fOnblz7uzunHtmfnMmMhNJkiRJUvfXq+oAJEmSJEkdwwJPkiRJknoICzxJkiRJ6iEs8CRJkiSph7DAkyRJkqQewgJPkiRJknoIC7xuLCIei4gtq46jahFxekR8v5PXeV5E/KQz11kvEXFgRPxzPpf1b1CSOkl799dV5MWOFBEHR8S/ah5nRKxSZUxSd2KB10Ei4oWImBoRb0XEq2UBsEg915mZa2XmbfVcR1fTfKcPkJlHZuaPq4qpShFxYkRc+EFeIzMvysxt27Gu9xW1jfg32KT8W8yI2KeF6f9qYf4XImLrmscbRcTfIuKNiJgQEf+OiEPmsq7BEXFNRLxcrnOFNmJbISJujYgpEfFk7XrL5w+IiBcj4u2IuCoilpinNy/pfZp9DngtIs7t6M8B7d1fN3Je7MrMG+osFngda5fMXARYD1gfOL7acOZdRPRpxHVXyW3e8SKidyes5vPAhPL7PImITYFbgNuBVYAPAUcBO8xlkdnAP4A927mKi4H/lq/7XeDyiFiyXPdawBnA54ClgCnAH+f1PUhqUdPngA2AjwHfaz5DT9rv9rD3Yt4wb/QcmelXB3wBLwBb1zw+Gbi+5vEmwN3AG8BDwJY1zy0BnAu8DEwErqp5bmfgwXK5u4F1mq8TWAaYCixR89z6wDigb/n4UOCJ8vVvAJavmTeBLwPPAM/P5f3tCjxWxnEb8JFmcRwPPF6+/rnAAvPwHr4NPAxMB/oAxwHPAZPL19yjnPcjwDRgFvAW8EY5/TzgJ+XPWwJjgG8AY4FXgENq1vch4FpgEvAf4CfAv1r5vW5e83sbDRxcs87TgOvLOO8DVq5Z7vfl/JOA+4Etap47EbgcuLB8/nBgI+Cecj2vAKcC/WqWWQu4kSIxvAZ8B9geeAeYUW6Ph8p5FwXOLl/nf+V77F0+dzBwF/Db8rV+Uk77V/l8lM+NBd4sfy8fBY4o1/NOua5rm//dA73LuJp+d/cDy87tNeeyvVv8X6iNsdnf7So1v48/AX8D3qb4UPVq0/su59kDeLj8uRfv/p2NB/5Czf9PO/7fl6dInnsCM4Glap57X6wtbKt/AafNx36mT/m+V2hlntUo/pcG1Ey7Eziy/PlnwIia51Yuf68D5jUev/zy690v3v854JfAdeXP78uztJ4blwWuBF4v91GnltPb3F+Xz51HmRfLx18AnqXY718DLFPzXAJHlrFNpMhtMZf3eCLvz19zzTk1636Cd3P6BuX0FnN98/dZE+Mqc4nJvNH6es0bDfjlGbw6iIihFEdUni0fD6EoBH5CsSP6JnBF05ER4AJgIYoP8R+m2GETERsA5wBfpChMzgCuiYj+tevLzJcpioPaozQHAJdn5oyI2J3ig/dngCUp/mkvbhb27sDGwJotvJ/Vyvm/Wi7/N+DaiOhXM9uBwHYU//SrUR61bOd72B/YCVgsM2dS7Dy3oEgaPwQujIjBmfkERRK6JzMXyczFmsdaWrpcdghwGHBaRCxePncaxY58aYojaHM9ihYRywF/B04p3/d6FMm4Nu4fAotT/K5/WvPcf8r5lwBGAJdFxAI1z+9GkSQXAy6iKFq/BgwCNgW2Ar5UxjEAuIniSNwyFEfubs7Mf1DsdC8tt8e65Wv/mSJ5rEJR6G9LkYSbbAyMovhbq42Zct5PUPwOFwP2BcZn5vAyzpPLde3Swib7erlNdgQGUhxUmDK312xheZjL/0I7HVC+nwHAryh+z59u9vyI8udjKP7mP0mxTZs+1LTXQcDIzLyC4oPLge1dMCIWovgdXz4P65sXawGjMnNyzbSHyulNzz/U9ERmPkeRqFerUzxSw4mIZSn2hf+tmbw7ZZ5tLTeWZ5KuA14EVqDIZZe0sJp27Vsj4tPAz4F9gMHl6zZ/vZ0pzjiuW863XStvr3n+mmvOiYi9KYrCgyjywq41MbaY61tZ79yYNz4480YPY4HXsa6KiMkUZ27GAieU0z8L/C0z/5aZszPzRmAksGO5M9uB4ijJxMyckZm3l8t9ATgjM+/LzFmZ+WeKIyybtLDuERQfromIAPbj3Z3SF4GfZ+YTZQH1M2C9iFi+ZvmfZ+aEzJzawmvvS3E28sbMnEGxE1wQ2KxmnlMzc3RmTqDYWe4/D+/hD+WyUwEy87LMfLncVpdSHFXcqIW45mYG8KNyW/6N4ozT6mXS3BM4ITOnZObjFIlpbg4EbsrMi8vXGp+ZD9Y8f2Vm/rvcphdRFHSU7+HCcv6ZmflroD+wes2y92TmVeV7nJqZ92fmveX8L1Ak+0+W8+4MvJqZv87MaZk5OTPvayngiFiK4u/pq5n5dmaOpUh2+9XM9nJmnlKuq/nvewZFoluD4gjuE5n5SivbqNbhwPcy86ksPJSZ49v7mm38L7TH1Zl5V7lNp1EclGj6nxhA8WGr6cDGF4HvZuaYzJxO8QFkr3kYbnQQ7/5/jWDehtssTrHvbe92nVeLUBzNr/Umxe+gPc9Lmn9XRcQbFGdbbqfIt01q82xruXEjigLi/8r9+LTMfN/1WbR/f30gcE5mPlDu744HNm12TdYvMvONzHwJuJWafNaCOfmLomhrLeccTnFg8D9lXng2M1+EDsn15o2OY97oYSzwOtbumTmAYpjgGhRnY6A4Lb93eVHsG+XOf3OKI2nLAhMyc2ILr7c88I1myy1LseNv7nKKHfYyFEf0kuJMXdPr/L7mNSZQDO0YUrP86Fbe1zIUR/wAKHfqo1tZ/sWaGNvzHt6z7og4KCIerJn/o7y7LdtjfFl0NZlCsXNakmKoQu36Wnvfy1IcYZybV1tYBwAR8Y2IeCIi3izfw6K89z00f8+rRcR1UTTomUTxoaBp/rbiqLU80Bd4pWb7nUFxVLPFddfKzFsohoeeBrwWEcMjYmA7191inPPwmq39L7RH8/c1AvhMebb4M8ADTR8sKLbTX2u20RMUZ1GXamslEfFxYEXePQI+Alg7ItYrH8+k+B0015fiA9lEimE683Okuj3eovjQVWsgxTCo9jwvaf7tnpmLZebymfmlZgfRavdRreXGZYEXm+Wx95mHfWvzHP4WxVm02hw+13zWgubvo7WcM9f81QG5vun1zRsfnHmjh7HAq4Py6NF5FGe6oNiBXFDu9Ju+Fs7MX5TPLRERi7XwUqOBnzZbbqHMbD68ksx8A/gnxdCKA4CLM4uB0uXrfLHZ6yyYmXfXvkQrb+llih0bMOcM4bIUY+2bLFvz83LlMu19D3PWXZ5VPBM4GvhQFsMwH6UoSNuKsy2vU+xEh84l7uZGUww5nScRsQXFdYX7AIuX7+FN3n0P8P738SfgSWDVzBxIMaS2af7W4mj+OqMpjgIPqtneAzNzrVaWee8LZv4hMzekGJKxGvB/7VmutThbec3my8/tf+FtiiE4AETE0i2tptk6H6f4ULMD7x1m07SuHZr9XS6QmbV/03PzeYrfzYMR8SrF9ZdQHJ0FeAlYrvw/aYp3IYoPPC9m5hTeP6S6Iz0GrFQefW6ybjm96fmm4bxExEoUZ5ifrlM8kgq1+6jWcuNoin1Im2eG2rlvbZ7DF6YYFtqe/V173kdrOafFvNCOXN9e5o2OYd7oYSzw6ud3wDbl0ZkLgV0iYruI6B0RC0TElhExtBxO8XfgjxGxeET0jYhPlK9xJnBkRGwchYUjYqdm/4C1RlDsLPbkvTul04Hjo+iCREQsWo6Lb6+/ADtFxFYR0Zeigcl0iovCm3w5IoZG0Tb3O8Cl8/keFqbY4b5exnoIxVG9Jq8BQ+O91/+1S2bOorho/cSIWCgi1uDdnWtLLgK2joh9IqJPRHyo5mhbawZQFJKvA30i4ge8/8hXS8tMAt4q4zqq5rnrgKUj4qtRXJ8xICI2Lp97DVghInqV7/EVikL/1xExMCJ6RcTKEfFJ2iEiPlb+rvpSJMempjZN61qplcXPAn4cEauWv+t1ym3W2mvO0cb/wkPAWhGxXhTXMp7YnvdD8X9wDMVZ7ctqpp8O/LT8kEFELBkRu7X1YuW696FoOrNezddXgAPLD2T3le/xuPJ/fWHgFxTDspuOBH8LODgi/i8iPlS+9roR0dJ1NrXrbrp2tX+895rOOTLzaYprRU8o178HsA5wRTnLRRT7oy3K2H5EMdzYI7FS52ktN/6bYijeL8rpC5RngN6jvftWiv3gIeX+sz/FCJH7srgc4ANpR845C/hmRGxYvs9Vyv1uW7l+XtZv3mh93eaNBmSBVyeZ+TpwPvD9zBxNcVHydyh2ZqMpjrI1bf/PUZyCf5Li2r2vlq8xkmKc/qkUp+efpei0NDfXAKsCr2Vm7cWwfwVOAi6JYvjfo8y9rW5L7+UpiusIT6HozLkLRSvod2pmG0Gxkx9Vfv1kft5DefTs1xRHql4D1qbo+tjkFoojSa9GxLj2vocaR1MMl3yV4sLsiymK1ZZieYli/P03KIa1PkjNEaxW3ECRcJ6m2DFPo/WhoFA03jmAYrjDmbxbIFPuQLeh2O6vUlyn8Kny6abkMz4iHih/Pgjox7tdTS+n/cM6Bpbrn1jGPp53z0SfTdEc4I2IuKqFZX9DcTDgnxTF6tkU12q29prNze1/4WmKhHITxftv6XqUllxMMWT6lsys/Xv5PcX/yz+juG72XormBwBEcR+rLVp4vd0pOtaen5mvNn2V77U3sH15bcZOvNvRdRTFEKl9ms6ql2fPP11+jYqICcBwigZGczOVYpgMFNtnztCvKG5qfHrNvPsBwyi2+S+Avcp9Epn5GEWzoosotvEAyoY+kjpHa7mxPBi5C0XTkpco9iP7tvAy7dq3ZubNwPcpPqy/QnFGbb/m830Ac805mXkZxXX5Iyjy21UUnSfbyvXzwrwxd+aNBhXvjuKT5k9EvAAcnpk3VR3LvIqIk4ClM3NeLnaWJEmSuiTP4KmhRMQa5dDBiIiNKG6j8Neq45IkSZI6Qntbu0o9xQCK4RfLUAwz+DVwdaURSZIkSR3EIZqSJEmS1EM4RFOSJEmSeggLPEmSJEnqIbrdNXiDBg3KFVZYoeowJEmd4P777x+XmUtWHUd3YY6UpMbQWn7sdgXeCiuswMiRI6sOQ5LUCSLixbbnUhNzpCQ1htbyo0M0JUmSJKmHsMCTJEmSpB7CAk+SJEmSeggLPEmSJEnqISzwJEmSJKmHsMCTJEmSpB7CAk+SJEmSeggLPEmSJEnqISzwJEmSJKmHsMCTJEmSpB6ibgVeRJwTEWMj4tG5PB8R8YeIeDYiHo6IDeoViyRJXYk5UpJUL/U8g3cesH0rz+8ArFp+HQH8qY6xSJLUlZyHOVKSVAd96vXCmXlHRKzQyiy7AednZgL3RsRiETE4M1+Z13Udcu6/ufWp1+c3VElSB5n19gwAei/clxd+sVPF0XRd9cqRL78xlR9c3eJJQUnz4Z2Zsxn31nQO2nQFALLmueLfk/dNr32QNQ+ydvp7FoAVBi3M0MUXnPO4VwT9+nglleZP3Qq8dhgCjK55PKac9r7kFRFHUBzBZLnllgMs6iSpK+q9cF+mPD2RBVdatOpQurv5ypELLL0y1z70cqcEKDWCiVOKg1Y3PTG209fdK2C9ZRebUyJmlrVjWR3muz+SZPH8nMfNCtB8t9ic8zrlPNm0QM1yObflEv73xlQAdll3mRaWf28c496azoOj3+Ab264+57l3Y8o5sTStpzb+2bOTx1+ZxD7DhtIroub9trxc7Xtumj5p6kw2XmkJgpjzPpZfYmEW7Ne7/b+IbqjKAi9amJYtTCMzhwPDAYYNG5YtFXefWn1Jzj1kow4PUpLUulmzZvPKK28xdOhAoEjK+sDmO0eO/MG29YxLaijTZszitqfGsuSA/jVT3/33jGhpKkTNE++dXjt/8eDWp8byr2fH8cnVliQCpr4zi1NueZaIYKF+fd67TBRLNU2LmnW9Oz3mrCvmLFesb85y8e76iZZf591l4933EHDlA/8D4NH/vUm0unzw1GuTATjpH08yv255suOL64M3W4FjtlqVJRbu1+Gv3RVUWeCNAZateTwUaNdhx6bizqJOkqr19tvv8NnP/pUHHniF++47nKWXXoRevVqqTTSP5jtHSuo4C/TtzfYfHVzXdaw9dFGO2WrV90z7xrar13WdH8Rv9llvnuafNmMWmbRYXEZNcTinMKypaO9+bhwD+vd937LFfDU/825xW/uaw+94jo+vMmjOfF+66AEAzrv7Bc67+wU+MnggHxk8gIfHvMmHB/TnjM9tyIAF+n6ArdM1VFngXQMcHRGXABsDb7bn+rsXxr3NIuXPFneSVJ1XXpnMrrtewsiRL7PYYgvw/PMTWXrpRdpeUO0xXzlSkrqaBfrO/3DIzVYe9IHW/fPPrPOexy/8YifufnYcJ/3jSR4a8yZPvDKJt6bPYPSEqTw79i3WPvGfc+Y9ea91WGfoosyanQxapD9LDVzgA8XSmepW4EXExcCWwKCIGAOcAPQFyMzTgb8BOwLPAlOAQ9rzupOnz2QRirN3kqRqPPLIa+y00whGj57Eiisuxt/+diBrrPHBEnEjqVeOlCS1brNVBnH10Zu/Z9rs2clK3/kbm670Ie4ZNR6Ab13+cIvLb7rSh7j4iE3qHucHEdm8jU8X13/wqjn487+zO5skVeSGG55l770vY/Lkd9h006FcffV+LLnkwnVZV0Tcn5nD6vLiPdCwYcNy5MiRVYchSd3adQ+/zNOvTmaNwQN5+rXJ/O6mZ1hi4X5MePudOfP8+ztb8eEKz+q1lh+rHKIpSepmRo2ayM47X8zMmbPZd9+1OPfc3Vhwwe5/vYIkSU12XmcZKEd37rj2YL669WpA0Vhm51P+BcBGP7sZgFU/vAg3fv2TlcQ5N95gQ5LUbiuttDg/+MEn+M53NmfEiD0t7iRJDeOjQxblmZ/uwHrLLjZn2jNj32KF466vLqgWeAZPktSqKVNm8NJLb865xu773+9aRyolSeosfXv34qovfxyA1ydP52M/vQkobgXx7e3XqDK0OTyDJ0maq1demcwnP3ken/rUn3nppTerDkeSpC5jyQH9WW6JhQD4023P8cfbnq04ooIFniSpRY888hobb3wWI0e+zIIL9mHq1BlVhyRJUpdyx7c+xQ92XhOAk//xFGfc/hxjJ02rNCYLPEnS+9xww7N8/OPnMHr0JDbddCj33Xc4q6/ubRAkSWru0M1XpHev4m7sP//7k2z0s5t5rcIizwJPkvQeZ5wxkp12GsHkye+w775rcfPNB9XtNgiSJPUEz/1sR+467tNzHm/8s5up6nZ0FniSpDkee2wsRx11PbNmpZ0yJUmaB0MWW5Dnf77jnMevT55eSRzdssD71OpLVh2CJPVIa631YX7zm+0455xd+elPt6JXOeREkiS1LSLYaIUlABh+x6hKYuiWBd65h2xUdQiS1GO8+upb3H//y3Mef/Wrm3DIIetXGJEkSd3XmZ8fBsBZ/3q+kvV3ywJPktQxHn10LBtvfBbbb38Rzz03oepwJEnq9hatubThzQo6UFvgSVKD+uc/n+PjHz+Hl156k1VXXYIBA/pXHZIkST3K9r+7o9PXaYEnSQ1o+PD72XHHi5g0aTr77FN0yvzwh+2UKUlSR3jmpzsA8MqbnX+7BAs8SWogs2cn//d//+SLX7xuTqfMiy+2U6YkSR2pb+93y6zOvl2CBZ4kNZAHHniF3/zmXvr06WWnTEmSOsGup97VqeuzwJOkBjJs2DIMH74zN9zwWTtlSpJUR3d+61MAPPK/Nzt1vX06dW2SpE736KNjmTBhKp/4xPIAHHbYBhVHJElSz7fsEgux7rKL8dDoNzp1vZ7Bk6QerKlT5m67XcIzz4yvOhxJkhpKU3E3bcasTlunBZ4k9VC1nTK32WYlhg4dWHVIkiQ1lKZ74p1267Odtk4LPEnqYZp3yjz++M255JK97JQpSVInu/zITQGYNbvzOml6DZ4k9SBTpszgs5+9kr/+9Un69OnFGWfszKGH2kxFkqQqrDCouMfsQv16d9o6LfAkqQd56KFXufbap1l00f5cccU+bLXVSlWHJElSw+vEE3gO0ZSknmTTTZdlxIjPcM89h1ncSZJUsaY7zf7mxqc7bZ2ewZOkbu7GG59j5szZ7LDDqgDsvfdaFUckSZIA+vTu/PNpnsGTpG5s+PD72WGHi9hnn8t57rkJVYcjSZKaWf5DCwEwc9bsTlmfBZ4kdUPNO2V+5SsbseKKi1cdliRJauadmUVh9+qkaZ2yPgs8SepmpkyZwd57X8avfnUPffr04qyzduFnP9uKXr2i7YUlSVKn+to2q3Xq+rwGT5K6kVdffYtdd72Y//znZTtlSpKk9/EMniR1Iy+88AYPP/waK6ywGHffbadMSZK6unueGw/A9Q+/0inrs8CTpG5kk02GcvXV+3HvvYex5ppLVh2OJElqw+c3WwGAn//9yU5Zn0M0JamLGz78fgYNWojPfOYjAGy33SoVRyRJktprvWUX69T1WeBJUhc1e3by7W/fyK9+dQ8LLtiHp5/+CkOHDqw6LEmSNI82XelDzOik2yRY4ElSFzRlygw+97m/cuWVT9CnTy9OOWUHiztJkrqpiOKrM1jgSVIXY6dMSZI0vyzwJKkLeeyxsey44wheeulNVlhhMa6//gCbqUiS1M3dXXbS7AwWeJLUhUybNpNx46bM6Zb54Q8vXHVIkiSpG/E2CZLUhWy44TLccstB3HLLQRZ3kiT1EF/acmX69u6ci/As8CSpQrNnJ9/61o1cdNHDc6ZtvPFQFlywb4VRSZKk7sohmpJUkdpOmQsv3JfttluFQYMWqjosSZLUjVngSVIFWuqUaXEnSZI+KAs8Sepkjz46lp12slOmJEnqeBZ4ktSJbr31eXbf/VImTZpup0xJktThbLIiSZ1oySWLYm7vvde0U6YkSepwnsGTpDrLTCKK1sgf/eiH+fe/D2fVVT9Er16d0y5ZkiQ1Ds/gSVIdTZkyg732uowzz7x/zrTVVx9kcSdJkurCAk+S6uTVV99iyy3P48orn+A737mFSZOmVx2SJEnq4RyiKUl18NhjRafMF198t1PmwIH9qw5LkiT1cBZ4ktTBbrzxOfba6zImTZrOxhsP4eqr92OppRapOixJktQAHKIpSR3okkseZYcdLmLSpOnsvfea3Hrr5y3uJElSp/EMniR1oA02GMyiiy7AEUdswE9/upXNVCRJUqeywJOkD+idd2bRr19vAFZb7UM8/viXPGsnSZLeY8asZOo7s1iw/MxQLw7RlKQP4LXX3mKLLc7lD3+4b840iztJklSrdzmi57nX36r7ujyDJ0nzqbZT5vjxU/jCFzZgwQX7Vh2WJEnqYtYesminrauuZ/AiYvuIeCoino2I41p4ftGIuDYiHoqIxyLikHrGI0kd5aabRrHZZufw4otvsvHGQ7jrrkMt7tRu5kdJUr3UrcCLiN7AacAOwJrA/hGxZrPZvgw8npnrAlsCv46IfvWKSZI6wllnPWCnTM0386MkqZ7qOURzI+DZzBwFEBGXALsBj9fMk8CAiAhgEWACMLOOMUnSB/KHP9zHscf+A4Djjvu4nTIbXEQMA7YAlgGmAo8CN2XmhFYWMz9KkuqmnkM0hwCjax6PKafVOhX4CPAy8AhwbGbObv5CEXFERIyMiJH1ClaS2mPXXVdnyJABnHXWLvz851tb3DWoiDg4Ih4AjgcWBJ4CxgKbAzdGxJ8jYrm5LN5h+bGMZU6OfP311+f7PUmSeoZ6nsFr6VNPNnu8HfAg8GlgZYqkeGdmTnrPQpnDgeEA/Qev2vw1JKmu3nhjGosu2p+IYIUVFuPpp7/CQgt5vV2DWxj4eGZObenJiFgPWBV4qaWnW5g2X/kR3psjhw0bZo6UpAZXzzN4Y4Blax4PpTgSWesQ4MosPAs8D6xRx5gkaZ489thY1lvvdE466a450yzulJmnza24K59/MDNvnsvT5kdJalCvvzW97uuoZ4H3H2DViFixvDB8P+CaZvO8BGwFEBFLAasDo+oYkyS1W22nzKuvfooZM2ZVHZK6mIhYMiK+ExHDI+Kcpq82FjM/SlKDWbTstD16wpS6r6tuQzQzc2ZEHA3cAPQGzsnMxyLiyPL504EfA+dFxCMUQ1a+nZnj6hWTJLXXWWc9wFFHXc/MmbPZa681Of/83enbt3fVYanruRq4E7gJaNcRAPOjJDWeFZdcGICZs+o/kr6uNzrPzL8Bf2s27fSan18Gtq1nDJI0L2bPTo4//iZOPvluwE6ZatNCmfnteV3I/ChJjaV/7+Ig8V3PjuPQzVes67rqeqNzSepuvve9Wzj55Lvp06eXnTLVHtdFxI5VByFJ6toWLa/fX3zh+t/S1AJPkmp8+csfY621luTvfz+Qww7boOpw1PUdS1HkTYuIyeXX+zpdSpI0ZLEFO2U9dR2iKUndwUsvvcmyyw4kIhgyZCAPPXQkvXt7/Etty8wBVccgSVItP8FIamg33TSKtdf+EyeccNucaRZ3mhcRsWtE/Kr82rnqeCRJjc1PMZIa1llnPcAOO1zEpEnTeeKJccye7T2iNW8i4hcUwzQfL7+OLadJklQJh2hKajjNO2V++9sf52c/s1Om5suOwHqZORsgIv4M/Bc4rtKoJEkNywJPUkOZOnUGn/vcX7niiifo3Tv405924gtf2LDqsNS9LQZMKH9etMI4JEmywJPUWL7+9Ru44oonGDiwP5dfvjfbbLNy1SGpe/s58N+IuJXihuSfAI6vNiRJUiOzwJPUUE48cUuefHI8p566A2ut9eGqw1E3l5kXR8RtwMcoCrxvZ+ar1UYlSWpkNlmR1OM98MArcxqoLLXUItx66+ct7vSBRMQa5fcNgMHAGGA0sEw5TZKkSngGT1KPdtZZD3DUUdfz1a9uzC9/uW3V4ajn+DpwBPDrFp5L4NOdG44kSQULPEk9UvNOmb16BZlJhJ0y9cFl5hHl909VHYskSbUcoimpx5k6dQb77HMZJ598N717B8OH78xJJ21jcacOFxF7R8SA8ufvRcSVEbF+1XFJkhqXBZ6kHuW1195iyy3/PKdT5j/+8Vlvg6B6+n5mTo6IzYHtgD8Dp1cckySpgVngSepRvvnNG/n3v//H8ssvyt13H8rWW69UdUjq2WaV33cC/pSZVwP9KoxHktTgvAZPUo/y+99vz+zZyW9+sy1LLbVI1eGo5/tfRJwBbA2cFBH98eCpJKlCJiFJ3d7f/vYMM2fOBmCJJRbkoos+Y3GnzrIPcAOwfWa+ASwB/F+lEUmSGpoFnqRua/bs5LjjbmKnnUZw7LF/rzocNZCIGFj+uABwGzA+IpYApgMjq4pLkiSHaErqlqZOncFBB13F5Zc/Tu/ewfrrD646JDWWEcDOwP0U972rbdGagBd/SpIqYYEnqdt57bW32G23S7jvvv8xcGB/rrhiH5upqFNl5s7l9xWrjkWSpFoO0ZTUrTz++OtsssnZ3HefnTJVvYjYIyIWrXm8WETsXmFIkqQGZ4EnqVv54Q9v54UX3mCjjYZw332Hs9ZaH646JDW2EzLzzaYHZaOVE6oLR5LU6ByiKalbOfPMXVh++UU58cQtWWihvlWHI7V0oNTcKkmqjGfwJHVps2cnw4ffzzvvFPeTHjiwPyefvI3FnbqKkRHxm4hYOSJWiojfUjRekSSpEhZ4krqsqVNnsO++l/PFL17Hl750fdXhSC35CvAOcCnwF2Aq8OVKI5IkNTSHkUjqkpp3ytxvv49WHZL0Ppn5NnBcRCySmW9VHY8kSZ7Bk9Tl2ClT3UVEbBYRjwOPl4/XjYg/VhyWJKmBWeBJ6lJuumkUm212tp0y1V38FtgOGA+QmQ8Bn6g0IklSQ7PAk9SlnHHG/bz55nT23PMj3Hrr51lqqUWqDklqVWaObjZpViWBSJKE1+BJ6mLOO283tthiOY4+eiN69Yqqw5HaMjoiNgMyIvoBxwBPVByTJKmBeQZPUqWmTp3BiSfextSpMwBYeOF+HHPMxhZ36i6OpOiaOQQYA6yHXTQlSRXyDJ6kyowd+za77XYJ9947hjFjJnHWWbtWHZLUbhHRG/hdZh5YdSySJDWxwJNUiccff52ddhrBCy+8wfLLL8rXvrZJ1SFJ8yQzZ0XEkhHRLzPfqToeSZLAAk9SBW6+eRR77vkX3nxzOhttNIRrrtnPZirqrl4A7oqIa4C3myZm5m8qi0iS1NAs8CR1qrPPfoAjj7yemTNns+eeH+H88/dgoYX6Vh2WNL9eLr96AQMqjkWSJAs8SZ0nM7nppueZOXM23/rWZvz851vbTEXdVkSsDzwGPJaZds6UJLVqdiaX3z+GX+29bl3XY4EnqdNEBOeeuxt7770mn/nMR6oOR5pvEfED4LPA/cDJEfHzzDyz4rAkSV3YMVutyugJU+q+Hm+TIKmuxo59myOOuJa33ip6UCywQB+LO/UE+wLrZeb+wMeAIyqOR5LUxe2/0XJ8a/s16r4ez+BJqpvaTpm9egWnn75z1SFJHWVaZk4ByMzxEeEBU0lSl2CBJ6kumnfK/OEPt6w6JKkjrVx2zgSIZo/JTG/qKEmqhAWepA5np0w1gN2aPf5VJVFIktSMBZ6kDjN7dvLd797ML35xF4CdMtVjZebtVccgSVJLvGZAUoeJgFdffZvevYPhw3fmpJO2sbhTjxQR10bELhHxvlPTEbFSRPwoIg6tIjZJUmPzDJ6kDhMRnHHGznzxixuyySZDqw5HqqcvAF8HfhcRE4DXgQWAFYDngFMz8+rqwpMkNSrP4En6QJ544nV22eVi3nxzGgD9+vW2uFOPl5mvZua3MnNlYG/gxxQF30czcxuLO0lSVSzwJM23m28exaabns111z3ND3/oJUlqTJn5Qmbek5kPNt06QZKkqljgSZovZ5/9ANtvfxFvvjmdPfZYg5/85NNVhyRJktTwLPAkzZPZs5Pjj7+Jww+/lpkzZ/PNb27K5Zfv420QJEmSugCbrEhqt5kzZ3PAAVdw2WWP07t3cNppO/LFLw6rOiypUhGxILBcZj5VdSySJHkGT1K79e4dLLnkQgwY0I/rrz/A4k4NLyJ2AR4E/lE+Xi8irqk0KElSQ7PAk9SmzASK2yD8/vc78MADX2S77VapOCqpSzgR2Ah4AyAzH6S4VYIkSZWwwJPUqptvHsXGG5/F+PFFc8A+fXqxyipLVByV1GXMzMw3qw5CkqQmFniS5qqpU+Z//vMyp57676rDkbqiRyPiAKB3RKwaEacAd1cdlCSpcdW1wIuI7SPiqYh4NiKOm8s8W0bEgxHxWER4Iy2pC2ipU+b3v//JqsOSuqKvAGsB04GLgUnAV9tayPwoSaqXunXRjIjewGnANsAY4D8RcU1mPl4zz2LAH4HtM/OliPhwveKR1D5Tp87g85+/ak6nzD/+cSeOOGLDqsOSuqTyxubfLb/axfwoSaqnet4mYSPg2cwcBRARlwC7AY/XzHMAcGVmvgSQmWPrGI+kNkybNpNPf/p87r13DAMH9ueyy/Zm221XrjosqcuKiFuBbD49Mz/dymLmR0lS3bSrwIuIj2bmo/P42kOA0TWPxwAbN5tnNaBvRNwGDAB+n5nnt7D+I4AjAPotbec+qV4WWKAPm2++LC+/PJnrrz+Aj37UkwZSG75Z8/MCwJ7AzDaW6bD8CO/Nkcstt1y7A5ck9UztPYN3ekT0A84DRmTmG+1YJlqY1vwoZx9gQ2ArYEHgnoi4NzOffs9CmcOB4QD9B6/6viOlkj6Y6dNn0r9/sTs46aRt+Pa3N2fQoIUqjkrq+jLz/maT7mrH9XIdlh/LGObkyGHDhpkjJanBtavJSmZuDhwILAuMjIgREbFNG4uNKedvMhR4uYV5/pGZb2fmOOAOYN12RS6pQ5x99gOsvfafeO21twDo1Sss7qR2ioglar4GRcR2wNJtLGZ+lCTVTbu7aGbmM8D3gG8DnwT+EBFPRsRn5rLIf4BVI2LF8uzffsA1zea5GtgiIvpExEIUQ1SemNc3IWne1XbKfOaZCVxxhf960ny4HxhZfr8H+AZwWBvLmB8lSXXT3mvw1gEOAXYCbgR2ycwHImIZioR2ZfNlMnNmRBwN3AD0Bs7JzMci4sjy+dMz84mI+AfwMDAbOGs+rvWTNI/slCl1jMxccT6WMT9KkuomMtserh8RdwBnApdn5tRmz30uMy+oU3zv03/wqjn9lWc6a3VSjzN27NvsttsldspUtxAR92fmsKrjaK6V0SsAZOb7Dnx2hmHDhuXIkSOrWLUkqRO1lh/b22TlyuZFXEQcm5m/78ziTtIHM3nydDbZ5Cyef/4NlltuUTtlSvNvl1aeS1oY2SJJUmdob4F3EPC7ZtMOBn7fkcFIqq8BA/rzuc+tw9///izXXLM/Sy+9SNUhSd1SZh5SdQySJLWk1QIvIvanuNnqihFRewH4AGB8PQOT1HEmTpzK4osvCMCJJ27J8cdvwQILtPf4jqTWRMROwFoU98EDIDN/VF1EkqRG1tYnvLuBV4BBwK9rpk+muPBbUhc2e3byve/dwp///BD33Xc4Q4cOJCIs7qQOEhGnAwsBnwLOAvYC/l1pUJKkhtbqp7zMfBF4Edi0c8KR1FGad8q8994x7LXXmlWHJfU0m2XmOhHxcGb+MCJ+jdffSZIq1NYQzX9l5uYRMZniovE5TwGZmQPrGp2k+VLbKXPAgH5cfvk+dsqU6qOps/SU8tZB44F5vnWCJEkdpa0zeJuX3wd0TjiSPqgnnnidnXYaYadMqXNcFxGLAb8EHqA4GHpmpRFJkhpae290vgnwWGZOLh8vAqyVmffVMzhJ82bChKl8/OPnMHHiNIYNW4Zrr7VTplQPEXE9MAL4TWa+DVwREdcBC2Tmm9VGJ0lqZL3aOd+fgLdqHk8pp0nqQpZYYkGOO25z9thjDW6//WCLO6l+hgM7A89HxKURsTvFpQsWd5KkSrW3lV5k5pxr8DJzdkTYhk/qAmbPTkaPfpPll18MgP/7v83IhF69otrApB4sM68Gro6IBYFdgc8Dp0fE34CLM/PGSgOUJDWs9p7BGxURx0RE3/LrWGBUPQOT1LapU2dwwAFX8LGPncnzz08EICIs7qROkplTM/PSzNwD2BZYH/hHxWFJkhpYewu8I4HNgP8BY4CNgSPqFZSktr3++ttstdX5XHrpY0ybNpMXX3RkmNTZImKpiPhKRNwFXAX8E9iw2qgkSY2sXcMsM3MssF+dY5HUTnbKlKoVEV8A9gdWp7jv3bcy865qo5Ikqe374H0rM0+OiFN4733wAMjMY+oWmaQW3XLL8+y551944w07ZUoV2gz4BXBTZs6uOhhJkpq0dQbv8fL7yHoHIqltr7wymZ12GsG0aTPZY481uPDCz7DQQn2rDktqOJl5SNUxSJLUkrYKvH2B64DFMvP3nRCPpFYMHjyAX/5yG1588Q1OOmkbm6lIkiTpPdoq8DaMiOWBQyPifOA9nyYzc0LdIpMEwLRpM3n66fGss85SABx99EYVRyRJkqSuqq0C73SKds8rAffz3gIvy+mS6uT1199mt90u4cknx3HPPYex+uqDqg5JUjMRsTmwamaeGxFLAotk5vNVxyVJakyt3iYhM/+QmR8BzsnMlTJzxZovizupjp544nU23vgs7rlnDAMG9GfGDPs4SF1NRJwAfBs4vpzUF7iwuogkSY2urS6aAzNzEvDdiFii+fMO0ZTqw06ZUrexB8XNzR8AyMyXI2JAtSFJkhpZW0M0RwA7UwzPTByiKdXduef+lyOOuI6ZM2fbKVPq+t7JzIyIBIiIhasOSJLU2Fot8DJz5/L7ip0TjtTYRo2aOKe4++Y3N7VTptT1/SUizgAWK29+fihwZsUxSZIaWFtn8ACIiD2AWzLzzfLxYsCWmXlV/UKTGs9KKy3OGWfszIwZs/jiF4dVHY6kNmTmryJiG2ASsDrwg8y8seKwJEkNrF0FHnBCZv616UFmvlFeWH5VXaKSGsjrr7/NU0+NZ/PNlwPg0EPXrzgiSe0VEV8DLrOokyR1Fa120WxjvvYWh5Lm4sknx7HJJmez444X8cgjr1UdjqR5NxC4ISLujIgvR8RSVQckSWps7S3wRkbEbyJi5YhYKSJ+S9F4RdJ8uuWW59l007MZNWoiq68+iEGDFqo6JEnzKDN/mJlrAV8GlgFuj4ibKg5LktTA2lvgfQV4B7gU+AswlSKZSZoP5577X7bb7kLeeGMau+++Brfd9nkGD7azutSNjQVeBcYDH644FklSA2vXMMvMfBs4LiIWycy36hyT1GPNnp18//u38LOf/QuAb3xjU046aWt6927vsRZJXUlEHAXsCywJXA58ITMfrzYqSVIja28Xzc2As4BFgOUiYl3gi5n5pXoGJ/U0Tz45jl/96h569w5OPXVHjjzSTplSN7c88NXMfLDqQCRJgvY3SvktsB1wDUBmPhQRn6hbVFIPteaaS3L++buz6KILsP32q1QdjqT5FBEDM3MScHL5eIna5zNzQiWBSZIaXrs7YWbm6Ij33HB5VseHI/U8Tz45jhdeeGNOQbfvvh+tOCJJHWAEsDNFw7EEahNkAitVEZQkSe0t8EaXwzQzIvoBxwBP1C8sqWe45Zbn2XPPv/DOO7O4++5DWXfdpasOSVIHyMydy+8rVh2LJEm12tvZ4UiKrplDgP8B62EXTalVtZ0yt912ZVZZZYm2F5LUrUTEze2ZJklSZ2lvF81xwIF1jkXqEeyUKfV8EbEAsBAwKCIW590hmgMp7ocnSVIl2ttFcyXg98AmFNcW3AN8LTNH1TE2qduZOnUGhxxyNZde+pidMqWe7YvAVymKuft5t8CbBJxWUUySJLX7GrwRFAlrj/LxfsDFwMb1CErqrp55ZgLXXPMUAwb047LL9ma77eyUKfVEmfl74PcR8ZXMPKXqeCRJatLeAi8y84KaxxdGxNH1CEjqztZZZykuu2xvlltuUdZee6mqw5FUf7MjYrHMfAOgHK65f2b+sdqwJEmNqr0XBd0aEcdFxAoRsXxEfAu4PiKWaH7vH6nR3HLL81x22WNzHu+002oWd1Lj+EJTcQeQmROBL1QXjiSp0bX3DN6+5fcjyu9N1xocivf7UQM799z/csQR19G7d/CRjyzJRz/64apDktS5ekVEZGYCRERvoF/FMUmSGlirBV5EfAwY3XSfn4j4PLAn8AJwYmZOqHuEUhfUvFPmscduykc+MqjiqCRV4AbgLxFxOsUBzyOBf1QbkiSpkbV1Bu8MYGuAiPgE8HPgKxT3wRsO7FXP4KSuaNq0mRx88FV2ypQE8G2KjppHUYxu+SdwVqURSZIaWlsFXu+as3T7AsMz8wrgioh4sK6RSV3Q66+/zW67XcI994yxU6YkMnM28KfyS5KkyrVZ4EVEn8ycCWzFu9fgtWdZqcd59dW3ePTRsSy77ECuv/4Am6lIDSoi/pKZ+0TEIxRDM98jM9epICxJktos0i4Gbo+IccBU4E6AiFgFeLPOsUldztprL8V11x3AqqsuweDBA6oOR1J1ji2/71xpFJIkNdNqgZeZP42Im4HBwD+buoRR3F7hK/UOTuoKzjvvQQAOPng9AD7xieWrC0ZSl5CZr5TfX6w6FkmSarU5zDIz721h2tP1CUfqOmo7Zfbp04vNN1+OVVbxto+SICIm08LQzCaZObATw5EkaQ6vo5Na0LxT5imn7GBxJ2mOzBwAEBE/Al4FLqDoonkg4PhtSVJlLPCkZuyUKWkebJeZG9c8/lNE3AecXFVAkqTG1qvqAKSu5KmnxrHJJmdzzz1jWHbZgdx116EWd5JaMysiDoyI3hHRKyIOBGZVHZQkqXFZ4Ek1MmHChKlsuOFg7rvvcG+DIKktBwD7AK+VX3uX0yRJqoRDNKUaa6wxiFtuOYjVVvsQCy/cr+pwJHVxmfkCsFvVcUiS1MQzeGpomUWnzNNO+/ecaeuvP9jiTlK7RMRqEXFzRDxaPl4nIr5XdVySpMZlgaeGNW3aTA444Ep+8pM7+frX/8mYMZOqDklS93MmcDwwAyAzHwb2qzQiSVJDq2uBFxHbR8RTEfFsRBzXynwfi4hZEbFXPeORmrz++ttstdX5XHLJowwY0I+rr96PoUO9bZWkebZQZv672bSZbS1kfpQk1UvdrsGLiN7AacA2wBjgPxFxTWY+3sJ8JwE31CsWqdZTT41jxx1HMGrURJZddiDXX3+AzVQkza9xEbEy5U3Py0LsldYWMD9KkuqpnmfwNgKezcxRmfkOcAktX4j+FeAKYGwdY5EAuOuul9h007MZNWqinTIldYQvA2cAa0TE/4CvAke2sYz5UZJUN/XsojkEGF3zeAxQezNYImIIsAfwaeBjdYxFAmDw4AH06dOL3Xdfgwsv3MNmKpLmW3mG7ajM3DoiFgZ6ZebkdixqfpQk1U09C7xoYVo2e/w74NuZOSuipdnLF4o4AjgCoN/S3nRa8yaz+LOLCFZaaXHuuecwVlhhMXr3tseQpPlX5q4Ny5/fnodFOyw/wntz5HLLLTcPYUiSeqJ6FnhjgGVrHg8FXm42zzDgkjJ5DQJ2jIiZmXlV7UyZORwYDtB/8KrNk6A0V9OmzeSQQ65mww0H881vbgbAyisvUXFUknqQ/0bENcBlwJwiLzOvbGWZDsuP5brm5Mhhw4aZIyWpwdWzwPsPsGpErAj8j6Jt9AG1M2Tmik0/R8R5wHUtJS9pfrz++tvsvvul3H33aP7+92c4+OD1GDRooarDktSzLAGMpxhK2SSB1go886MkqW7qVuBl5syIOJqi+1dv4JzMfCwijiyfP71e65Za6pRpcSepI0XEkhTdMJ/NzDfau5z5UZJUT9F0fVJ30X/wqjn9lWeqDkNd2G23vcBnPnMpEydOY8MNB3PttfszePCAqsOSNB8i4v7MHFZ1HM1FxOHAz4DngBWBIzLzmmqjKoZojhw5suowJEl11lp+rOcQTanTXXXVk+yzz2XMmDGb3XZbnYsu+oydMiXVw1eBtTLz9YhYCbgIqLzAkyTJAk89yoYbDmbQoIXYf/+PcvLJ29gpU1K9vJOZrwNk5qiI6F91QJIkgQWeeoB33plF3769iAiWXXZRHn74KK+3k1RvQyPiD3N7nJnHVBCTJEkWeOreXn/9bfbY41K22WYlTjhhSwCLO0md4f+aPb6/kigkSWrGAk/dVm2nzJdeepOvfW1TBg50lJSk+svMP1cdgyRJLfECJXVLt932AptuejajRk1kgw0Gc++9h1vcSZIkqeFZ4Knb+fOfH2TbbS9g4sRp7Lrr6txxx8Ess4y3QZAkSZIs8NStnHnm/Rx88NXMmDGbr31tE668ch9vgyBJkiSVLPDUreyyy+qstNLinHbajvzmN9t5GwRJlYqI1SLi5oh4tHy8TkR8r+q4JEmNy0/H6vImTpzK7NkJwNJLL8Jjj32JL33pYxVHJUkAnAkcD8wAyMyHgf0qjUiS1NAs8NSlPfXUOIYNO5PvfOfmOdMWWMDmr5K6jIUy89/Nps2sJBJJkrDAUxdW2ynzxhtHMXXqjKpDkqTmxkXEykACRMRewCvVhiRJamSeClGX9Oc/P8gXvnAtM2bMZtddV2fEiM+w4IJ9qw5Lkpr7MjAcWCMi/gc8DxxYbUiSpEZmgacuJTP5wQ9u5Sc/uROAr31tE375y21spiKpq3oxM7eOiIWBXpk5ueqAJEmNzU/N6lJ+/vN/8ZOf3EmvXsEf/2inTEld3vMRMRzYBHir6mAkSfKTs7qUI47YkPXXX5rrrtufo46yU6akLm914CaKoZrPR8SpEbF5xTFJkhqYQzRVuRdeeIOhQwfSp08vBg1aiJEjj6BXr6g6LElqU2ZOBf4C/CUiFgd+D9wO9K40MElSw/IMnip1220vsMEGZ/D1r98wZ5rFnaTuJCI+GRF/BB4AFgD2qTgkSVID8wyeKlPbKfPFF99kxoxZ9O3rQW9J3UdEPA88SHEW7/8y8+1qI5IkNToLPHU6O2VK6kHWzcxJVQchSVITCzx1qmnTZnLooVdz8cWP0qtXcOqpO9hMRVK3ExHfysyTgZ9GRDZ/PjOPqSAsSZIs8NS5fvCDW7n44kdZZJF+/OUve7HDDqtWHZIkzY8nyu8jK41CkqRmLPDUqb7znS14+OHXOOmkrVl33aWrDkeS5ktmXlv+OCUzL6t9LiL2riAkSZIAu2iqE9x//8vMmDELgMUWW4B//OOzFneSeorj2zlNkqRO4Rk81dX55z/E4Ydfw0EHrcuZZ+5ChLdAkNT9RcQOwI7AkIj4Q81TA4GZ1UQlSZIFnuokMznhhNv48Y/vAGDgwP5kgvWdpB7iZYrr73YF7q+ZPhn4WiURSZKEBZ7qoHmnzFNO2YEvfclOmZJ6jsx8CHgoIi7KTM/YSZK6DAs8dajXX3+bPfa4lLvuGm2nTEk9VkT8JTP3Af7b7DYJAWRmrlNRaJKkBmeBpw514om3cdddoxk6dCDXXbe/zVQk9VTHlt93rjQKSZKascBThzrppG2YMmUmP/3pp1lmmQFVhyNJdZGZr5Q/jgOmZubsiFgNWAP4e3WRSZIanbdJ0Ad23XVPM21acQnKIov049xzd7O4k9Qo7gAWiIghwM3AIcB5lUYkSWpoFniab0WnzFvZZZeLOfTQq8nMtheSpJ4lMnMK8BnglMzcA1iz4pgkSQ3MIZqaL9OmzeSww65hxIhH6NUr2Hzz5bzHnaRGFBGxKXAgcFg5zdwqSaqMSUjzbNy4Key++yV2ypQk+CpwPPDXzHwsIlYCbq02JElSI7PA0zx56qlx7LTTCJ57bqKdMiU1vMy8Hbg9IgZExCKZOQo4puq4JEmNy2vwNE9+9au7ee65iWywwWDuu+9wiztJDS0i1o6I/wKPAo9HxP0RsVbVcUmSGpdn8DRP/vCHHVhyyYX57ne3YOGF+1UdjiRV7Qzg65l5K0BEbAmcCWxWYUySpAbmGTy1KjM5/fSRvP32OwAsuGBffvazrSzuJKmwcFNxB5CZtwELVxeOJKnRWeBprqZNm8lnP/tXjjrqeg4++Oqqw5GkrmhURHw/IlYov74HPF91UJKkxmWBpxaNGzeFbba5gBEjHmGRRfpxyCHrVR2SJHVFhwJLAleWX4MobnYuSVIlvAZP7/P00+PZcceL7JQpSXMREQsARwKrAI8A38jMGdVGJUmSBZ6auf32F9hjj0uZOHEaG2wwmGuv3Z9llhlQdViS1NX8GZgB3AnsAHyE4p54kiRVygJP73HBBQ8zceI0dt11dUaM+IzNVCSpZWtm5toAEXE28O+K45EkCbDAUzOnnbYj6623NEcdNYzevb1EU5LmYs5wzMycGRFVxiJJ0hx+gm9w06bN5Hvfu4VJk6YD0L9/H44+eiOLO0lq3boRMan8mgys0/RzREyqOjhJUuPyDF4DGzduCrvvfgl33TWap54az2WX7V11SJLULWRm76pjkCSpJRZ4Daq2U+aQIQP43ve2qDokSZIkSR+QBV4Dqu2Uuf76S3PttfszZMjAqsOSJEmS9AF5oVWDueCCh9hmmwuYOHEau+yyGnfccYjFnSRJktRDWOA1mLvvHs2MGbM59tiN+etf92WRRbwNgiRJktRTOESzwZxyyo5su+3K7LHHR6oORZIkSVIH8wxeDzdu3BQOOeRqJk6cCkCfPr0s7iRJkqQeyjN4PVhtp8x33pnFRRd9puqQJEmSJNVRXc/gRcT2EfFURDwbEce18PyBEfFw+XV3RKxbz3gaye23v8Amm5zFc89NZP31l+bkk7euOiRJUsn8KEmql7oVeBHRGzgN2AFYE9g/ItZsNtvzwCczcx3gx8DwesXTSOyUKUldl/lRklRP9TyDtxHwbGaOysx3gEuA3WpnyMy7M3Ni+fBeYGgd4+nxMpMTTriVgw66yk6ZktR1mR8lSXVTzwJvCDC65vGYctrcHAb8vY7x9HgRweTJ79CrV3DKKTvwu99tT+/e9tGRpC7G/ChJqpt6NlmJFqZlizNGfIoigW0+l+ePAI4A6Lf0Kh0VX4/0y19uw777rsXGG3uwV5K6qA7Lj+U8c3Lkcsst1xHxSZK6sXqe3hkDLFvzeCjwcvOZImId4Cxgt8wc39ILZebwzByWmcPqEmk39vTT49luuwsZO/ZtAHr37mVxJ0ldW4flR3hvjlxyySU7PFhJUvdSzwLvP8CqEbFiRPQD9gOuqZ0hIpYDrgQ+l5lP1zGWHumOO15kk03O4p//fI7vfe+WqsORJLWP+VGSVDd1G6KZmTMj4mjgBqA3cE5mPhYRR5bPnw78APgQ8MeIAJjpWbr2ueCChzjssGuYMWM2u+yyGr/5zXZVhyRJagfzoySpniKzxWH/XVb/wavm9FeeqTqMymQmJ554Gz/60R0AHHvsxvz619vaTEVSjxQR91vYtN+wYcNy5MiRVYchSaqz1vJjPZusqIPNnp0cdNBfueiiR+jVK/j977fn6KM3qjosSZIkSV2EBV430qtXsMIKi7HIIv245JI92Wmn1aoOSZIkSVIXYoHXDcyenfTqVXTV/tGPPsWhh67PSistXnFUkiRJkroaL9zq4u6440U22OAMXn55MlCcxbO4kyRJktQSC7wu7IILHmLrrc/noYde43e/u7fqcCRJkiR1cRZ4XVBmcsIJt3LQQVcxY8Zsjj12Y37+862qDkuSJElSF+c1eF3M9OkzOeywa+yUKUmSJGmeWeB1ITNnzmbbbS/kjjtetFOmJEmSpHlmgdeF9OnTi+22W5nnnpvAddcdwHrrLV11SJIkSZK6Ea/B6wKmTZs55+fjj9+chx8+yuJOkiRJ0jyzwKvYBRc8xGqrncKLL74BQESwxBILVhuUJEmSpG7JAq8imcmJJ97GQQddxejRk7jssserDkmSJElSN+c1eBWwU6YkSZKkerDA62Tjxk1hjz0u5V//eomFF+7LpZfuZadMSZIkSR3CAq8TTZs2k803P4ennhrPkCED7JQpSZIkqUN5DV4nWmCBPhx11DDWX39p7rvvcIs7SZIkSR3KAq8TjB8/Zc7PxxyzMffccxhDhgysMCJJkiRJPZEFXh01dcpcffVTeeaZ8UBxG4T+/R0ZK0mSJKnjWWnUyfTpMzn88Gu58MKH6dUruPfeMay66oeqDkuSJElSD2aBVwfjxxedMu+8006ZkiRJkjqPBV4He+aZ8ey00wieeWaCnTIlSZIkdSoLvA40efJ0Nt/8XMaOfZv11lua667b32YqkiRJkjqNTVY60IAB/fnxjz/Fzjuvxp13HmJxJ0mSJKlTWeB9QJnJs89OmPP4iCM25Jpr9mORRfpVGJUkSZKkRmSB9wFMnz6Tgw66ig03HM5jj42dMz0iKoxKkiRJUqOywJtP48dPYZttLuDCCx9m1qzZjBkzqeqQJEmSJDU4m6zMh9pOmcssM4Drrtuf9dcfXHVYkiRJkhqcBd48uuOOF9ljj0uZMGEq6623NNdeuz9Dh9pMRZIkSVL1LPDmwYQJU9lppxG89dY77LTTqlxyyV42U5EkSZLUZVjgzYMllliQP/5xR/7zn5f57W+3o3dvL2GUJEmS1HVYobRh+vSZjBz58pzHn/vcuvzhDztY3EmSJEnqcqxSWtHUKXPLLc/jv/99pepwJEmSJKlVDtGci+adMiVJkiSpq7PAa8Gdd77I7rvbKVOSJElS9+IQzWYuvPBhtt76grJj5qrceechFneSJEmSugULvBqvvDKZI464lnfemcUxx2zE1Vfv520QJEmSJHUbDtGsMXjwAM4/fw9efnkyxxyzcdXhSJIkSdI8afgCb/z4KTz44KtstdVKAOy115oVRyRJkiRJ86ehh2g+88x4Nt30bHbe+WLuu29M1eFIkiRJ0gfSsAXenXe+yCabnM0zz0xgjTUGMWSIjVQkSZIkdW8NWeDZKVOSJElST9RQBV5m8sMf3sbnPvdXO2VKkiRJ6nEaqsnKCy+8wS9/eTe9egW//e12dsqUJEmS1KM0VIG34oqL85e/7M3s2cnOO69WdTiSJEmS1KF6fIH3zDPjefjh19hzz+L2BzvuuGrFEUmSJElSffToAu/OO19k990vZfLk6dx++wA23XTZqkOSJEmSpLrpsU1WLrro3U6Z2267MmuvvVTVIUmSJElSXfW4Aq+pU+ZnP1t0yvzKV+yUKUmSJKkx9KghmtOnz+Tww6/lwgsftlOmJEmSpIbTowq80aMnce21T7Hwwn255JK97JQpSZIkqaH0qAJvlVWW4Kqr9mPRRfuz/vqDqw5HkiRJkjpVty/w7rzzRZ59dgKHHLI+AFtuuUK1AUmSJElSRbp1gXfRRQ9z6KHXMGvWbNZa68NstNGQqkOSJEmSpMrUtYtmRGwfEU9FxLMRcVwLz0dE/KF8/uGI2KA9r9u8U+aXvvQxNtjAIZmSpO6hXvlRkqS6ncGLiN7AacA2wBjgPxFxTWY+XjPbDsCq5dfGwJ/K73OXcNBBV9kpU5LULdUtP0qSRH3P4G0EPJuZozLzHeASYLdm8+wGnJ+Fe4HFIqLVU3EzJkzjwgsfZuGF+3LVVfta3EmSupu65EdJkqC+Bd4QYHTN4zHltHmd5z1y5myWWWYAd955CLvssnqHBCpJUieqS36UJAnq22QlWpiW8zEPEXEEcET5cPrLL3/j0Q02+MYHDK9HGwSMqzqILszt0za3UevcPm3rqG20fAe8RlfTYfkR3p8jI+LRDxBbT+f/btvcRq1z+7TNbdS6uufHehZ4Y4Blax4PBV6ej3nIzOHAcICIGJmZwzo21J7FbdQ6t0/b3Eatc/u0zW3Uqg7Lj2COnBdun7a5jVrn9mmb26h1nbF96jlE8z/AqhGxYkT0A/YDrmk2zzXAQWW3sE2ANzPzlTrGJElS1cyPkqS6qdsZvMycGRFHAzcAvYFzMvOxiDiyfP504G/AjsCzwBTgkHrFI0lSV2B+lCTVU11vdJ6Zf6NIUrXTTq/5OYEvz+PLDu+A0Ho6t1Hr3D5tcxu1zu3TNrdRK+qUH8Ht3ha3T9vcRq1z+7TNbdS6um+fKHKIJEmSJKm7q+c1eJIkSZKkTtRlC7yI2D4inoqIZyPiuBaej4j4Q/n8wxGxQRVxVqUd2+fAcrs8HBF3R8S6VcRZpba2Uc18H4uIWRGxV2fGV7X2bJ+I2DIiHoyIxyLi9s6OsWrt+D9bNCKujYiHym3UUNdJRcQ5ETF2bm35G30/XS/mx7aZI1tnfmybObJ15sfWVZ4fM7PLfVFcdP4csBLQD3gIWLPZPDsCf6e4V9AmwH1Vx93Fts9mwOLlzzs00vZp7zaqme8Wimth9qo67q60fYDFgMeB5crHH6467i64jb4DnFT+vCQwAehXdeyduI0+AWwAPDqX5xt2P13HbW5+7Jht1LA50vzYYX9DDZsjzY/t2kaV5seuegZvI+DZzByVme8AlwC7NZtnN+D8LNwLLBYRgzs70Iq0uX0y8+7MnFg+vJfiHkqNpD1/QwBfAa4AxnZmcF1Ae7bPAcCVmfkSQGa6jd6/jRIYEBEBLEKRwGZ2bpjVycw7KN7z3DTyfrpezI9tM0e2zvzYNnNk68yPbag6P3bVAm8IMLrm8Zhy2rzO01PN63s/jOIoQSNpcxtFxBBgD+B0Gk97/oZWAxaPiNsi4v6IOKjTousa2rONTgU+QnED6keAYzNzdueE1y008n66XsyPbTNHts782DZzZOvMjx9cXffTdb1NwgcQLUxr3u6zPfP0VO1+7xHxKYrktXldI+p62rONfgd8OzNnFQeYGkp7tk8fYENgK2BB4J6IuDczn653cF1Ee7bRdsCDwKeBlYEbI+LOzJxU59i6i0beT9eL+bFt5sjWmR/bZo5snfnxg6vrfrqrFnhjgGVrHg+lOAIwr/P0VO167xGxDnAWsENmju+k2LqK9myjYcAlZfIaBOwYETMz86pOibBa7f0fG5eZbwNvR8QdwLpAIyQvaN82OgT4RRYD6p+NiOeBNYB/d06IXV4j76frxfzYNnNk68yPbTNHts78+MHVdT/dVYdo/gdYNSJWjIh+wH7ANc3muQY4qOxCswnwZma+0tmBVqTN7RMRywFXAp9rkKNJzbW5jTJzxcxcITNXAC4HvtRAyas9/2NXA1tERJ+IWAjYGHiik+OsUnu20UsUR2+JiKWA1YFRnRpl19bI++l6MT+2zRzZOvNj28yRrTM/fnB13U93yTN4mTkzIo4GbqDo1HNOZj4WEUeWz59O0dVpR+BZYArFkYKG0M7t8wPgQ8AfyyNwMzNzWFUxd7Z2bqOG1Z7tk5lPRMQ/gIeB2cBZmdliu9+eqJ1/Qz8GzouIRyiGW3w7M8dVFnQni4iLgS2BQRExBjgB6Avup+vF/Ng2c2TrzI9tM0e2zvzYtqrzYxRnTiVJkiRJ3V1XHaIpSZIkSZpHFniSJEmS1ENY4EmSJElSD2GBJ0mSJEk9hAWeJEmSJPUQFnhSGyLinIgYGxHz1f44InaOiP9GxEMR8XhEfLGD4zsrItYsf/5Os+fubmPZYRHxh/LnLSNis46MTZLUvUTEdyPisYh4OCIejIiNO/j17y6/rxARB9RMn5OPWln2yIg4qPz54IhYZj7Wf3lErFT+/EJEPFK+19sjYvl5fb021nVbRAyrWdegiOgXEXdERJe8VZl6Bgs8qW3nAdvPz4IR0RcYDuySmesC6wO3dVhkQGYenpmPlw+/0+y5Vgu2zByZmceUD7cELPAkqUFFxKbAzsAGmbkOsDUwuiPXUZOXVgAOqJlem4/mtuzpmXl++fBgYJ4KvIhYC+idmbU33P5U+V5vA743L683PzLzHeBmYN96r0uNywJPakNm3gFMmM/FBwB9gPHla03PzKcAImLJiLgiIv5Tfn28nH5iedbwtogYFRHHlNMXjojryzOBj0bEvuX028ojn78AFiyPuF5UPvdW+f3SiNixKaiIOC8i9izP2l0XESsARwJfK5ffIiKeLwtUImJgefSx73xuB0lS1zcYGJeZ0wEyc1xmvgwQERuWZ7nuj4gbImJwOf22iDgpIv4dEU9HxBbl9LXKaQ+WZ8hWLae/Va7rF8AW5fNfq8lHvcp8s1hTUBHxbEQsVebHb0bEXsAw4KJy+Z0i4q81828TEVe28P4OBK6ey3u/BxhSLj+3/LxIRJxbc9Zvz3L6nyJiZHnm84ft2M5XlbFIdWGBJ9VRZk4ArgFejIiLI+LAiGj6v/s98NvM/BiwJ3BWzaJrANsBGwEnlIXV9sDLmbluZn4U+EezdR0HTM3M9TKzeeK4hPJoYUT0A7YC/laz7AvA6WU862XmnRRHM3cqZ9kPuCIzZ8z/1pAkdXH/BJYtC7U/RsQnYc5olFOAvTJzQ+Ac4Kc1y/XJzI2ArwInlNOOBH6fmetRFGNjmq3rOODOMuf8tmliZs6mKML2KNe9MfBCZr5WM8/lwEjgwPL1/wZ8JCKWLGc5BDi3hff3ceD+ubz37SkKL5h7fv4+8GZmrl2e9bulnP7dzBwGrAN8MiLWmcs6mjwKfKyNeaT5ZoEn1VlmHk5RUP0b+CZFYoRi6MupEfEgRRE4MCIGlM9dX57tGweMBZYCHgG2Lo+UbpGZb85DGH8HPh0R/YEdgDsyc2oby5xFkSRh7slSktRDZOZbwIbAEcDrwKURcTCwOvBR4MYyZ30PGFqzaNPZsvsphl5CcUbsOxHxbWD5duScWpfy7hDG/crHrcWdwAXAZ8szf5tS5L3mBpfvq9atETGWIiePKKfNLT9vDZxWs96J5Y/7RMQDwH+BtYA124h3FvBOTc6XOpQXeEofUET05t0jgtdk5g+az5OZjwCPRMQFwPMU1w70AjZtnvQiAmB6zaRZFEdHn46IDYEdgZ9HxD8z80ftiTEzp0XEbRRnBfcFLm7HMndFcRH8JymuWZivJjOSpO6jLD5uA26LiEeAz1PkuMcyc9O5LNaUs2ZRfrbMzBERcR/FSJAbIuLwzLxlLss3dw+wSnlGbnfgJ+1Y5lzgWmAacFlmzmxhnqnAAs2mfQp4m+J6+x8BX2fu+TmAbDZtRYqDtx/LzIkRcV4L62hJ/zJWqcN5Bk/6gDJzVjnEZL3mxV05Xn/LmknrAS+WP/8TOLpm3vVaW08U3cKmZOaFwK+ADVqYbUYr18ldQnEmbgvghhaen0xxzWCt8ymKQc/eSVIPFxGrN10rV1qPImc9BSwZRRMWIqJvFA1LWnutlYBRmfkHirNgzYcttpRzgDln5P4K/AZ4IjPHtzDbe5YvrxV8meLs4nlzCesJYJUW1jeVYnjpQRGxBHPPz82nLw4MpCgQ34yIpShGybQqIj4EvO5lD6oXCzypDRFxMcXRxNUjYkxEHDYviwPfioinyqEeP6Q4ewdwDDCsvFD7cYrrFVqzNvDv8nW+S8tHNIcDD0fZZKWZfwKfAG4qu3g1dy2wR1OTlXLaRcDitOOMnySp21sE+HMUt/R5mGKo4YllztgLOCkiHgIepO2uy/sCj5Y5aw2KA4a1HgZmRtE47GstLH8p8FnmPjzzPOD0MmctWE67CBhd01m6uespOka/T2a+QpHrvszc8/NPgMWjaHT2EEUHzocohmY+RnEJxl1zWXetT1FzHbzU0aI4SCJJ71d2KtstMz9XdSySJLUmIk4F/puZZ8/l+QWBW4GPl0NRK1F2+Dy+qau21NG8Bk9SiyLiFIqhJju2Na8kSVWKiPsphkp+Y27zZObUiDiB4nYIL3VWbLXKTtZXWdypnjyDJ0mSJEk9hNfgSZIkSVIPYYEnSZIkST2EBZ4kSZIk9RAWeJIkSZLUQ1jgSZIkSVIPYYEnSZIkST3E/wMsnIXW5fxMigAAAABJRU5ErkJggg==\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, (roc_fig, prc_fig) = plt.subplots(nrows=1, ncols=2, figsize=(15, 6))\n",
    "aurocs = []\n",
    "auprcs = []\n",
    "for model_name, inferences in model_inferences.items():\n",
    "    labels = inferences['labels']\n",
    "    predictions = inferences['predictions']\n",
    "    fpr, tpr, ft_thresholds = roc_curve(y_true=labels, y_score=predictions)\n",
    "    positive_predictive_value, sensitivity, pr_thresholds = precision_recall_curve(labels, predictions)\n",
    "    optimal_roc_idx = np.argmax(tpr - fpr)\n",
    "    optimal_roc_threshold = ft_thresholds[optimal_roc_idx]\n",
    "    optimal_prc_idx = np.argmax(sensitivity + positive_predictive_value)\n",
    "    optimal_prc_threshold = pr_thresholds[optimal_prc_idx]\n",
    "    auroc = roc_auc_score(y_true=labels, y_score=predictions)\n",
    "    auprc = average_precision_score(y_true=labels, y_score=predictions)\n",
    "    aurocs.append(auroc)\n",
    "    auprcs.append(auprc)\n",
    "\n",
    "    prc_fig.step(sensitivity, positive_predictive_value, where='post')\n",
    "    roc_fig.plot(fpr, tpr, lw=2)\n",
    "    print(\"Optimal Threshold from ROC {:.2f}, optimal threshold from PRC {:.2f}\".format(optimal_roc_threshold, optimal_prc_threshold))\n",
    "\n",
    "\n",
    "roc_fig.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n",
    "roc_fig.set_xlim([0.0, 1.05])\n",
    "roc_fig.set_ylim([0.0, 1.05])\n",
    "roc_fig.set_xlabel('1 - Sensitivity')\n",
    "roc_fig.set_ylabel('Specificity')\n",
    "roc_fig.set_title('Receiver operating characteristics curve. AUC {:.2f}'.format(np.array(aurocs).mean()))\n",
    "\n",
    "prc_fig.set_xlabel('Sensitivity (Recall)')\n",
    "prc_fig.set_ylabel('Positive Predictive Value (Precision)')\n",
    "prc_fig.set_ylim([0.0, 1.05])\n",
    "prc_fig.set_xlim([0.0, 1.05])\n",
    "\n",
    "prc_fig.set_title(\"Precision recall curve. AUC {:.2f}\".format(np.array(auprcs).mean()))\n",
    "plt.savefig(os.path.abspath(os.path.expanduser(f\"~/catkin_ws/src/rt_gene/assets/rtbene_pytorch_resnet18_{mode}.png\")))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "data": {
      "text/plain": "Calculating F-Scores:   0%|          | 0/1000 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "b4f7edec278b40ff9b7cb340bd9bd8de"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimal Threshold from F1-Score Thresholding 0.99\n"
     ]
    },
    {
     "data": {
      "text/plain": "Text(0.5, 1.0, 'Optimal Threshold: 0.99')"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAjoElEQVR4nO3deXhc9X3v8fdX+2ZZqzd5kRcWm7AZA04gCQlJA4SUhuamJEAakpRw26TtbdpCtyxP0qdJmntv0zaE0pTQhlxoFpIAgbgkaYAWDMZgG4yxseVFsmzt+zaame/944zFSJbQGEYandHn9Tx6Hs2c38x8fyP7M7/5nd85x9wdEREJv5xMFyAiIumhQBcRyRIKdBGRLKFAFxHJEgp0EZEsoUAXEckSCnRJGzNbaWb9ZpY7A8/9eTO753U+9pCZvSvdNU3yOr8ys0+8zsdOWaOZXWZmTW+sOpkPFOjzmJl91MxeMLNBMztuZt80s4pTePy4EHL3I+5e5u6xGSl48hquT3yI9JvZkJnFk273z1Ydc4mZ1ZvZfyb+ri+/1oeZmVWY2b+aWWvi5/MTtr/FzJ4xsz4z22Vml854B+R1U6DPU2b2GeArwJ8AC4HNwCrgUTMryGRtp8Ldv5v4ECkDrgSaT9xO3HdKzCwv/VXOunuB54Fq4C+AH5hZ7RRt/y9QAtQDFwE3mtlNAGZWBTwA/C1QAXwVeNDMKmeyeHn9FOjzkJmVA18APu3uP3P3UXc/BHyQINRvSLT7vJn9wMz+PTFCe87Mzk1s+w6wkuA/eL+Z/WliZOgnQjExBfElM3sy0eZBM6s2s++aWa+ZbTOz+qS6vm5mjYlt283srWns9nmJEWZPoj9Fide8zMyazOxWMzsOfNvMcszsNjM7YGYdZva9RLhhZkVmdk/i/u5EHxYnvc4qM/vvxPv1H2ZWk9S/Xzez3YnH/crM1k9WqJkVm9ndZtZlZi8BF6baSTM7HdgIfM7dh9z9h8ALwG9O8ZD3AV9198HEv4F/AT6W2PYWoMXdv+/uMXe/B2gDrk21HpldCvT56S1AEXB/8p3u3g88Arw76e5rgO8DVcD/A35sZvnufiNwBHhfYjT81Sle6zrgRqAOWAs8BXw78Xx7gM8ltd0GnJf0Wt8/Ebxp8EHgCmA1cA7w0aRtSxKvuQq4Gfh94DeAtwPLgC7gG4m2v03wjWYFwQj4FmAo6bk+DNwELAIKgD+GsaC9F/hDoBZ4mODDcLJvQ58jeK/WAu9JvOYYM7vdzG6fop9nAQ3u3pd0387E/VOxCb+/Kel3m6Ttm5A5SYE+P9UA7e4enWTbscT2E7a7+w/cfRT4PwQfBJtP4bW+7e4H3L2H4MPigLv/PPHa3wfOP9HQ3e9x9w53j7r7/wYKgTNOrWtT+nt3b3b3TuBBgg+OE+IEI9oRdx8CPgn8hbs3ufsI8HngA4lvHqMEQb4uMWrd7u69E/q7L/E830t6nd8Cfurujybey68BxQQfrhN9EPhrd+9090bg75M3uvvvuvvvTtHPMqBnwn09wIIp2v8MuM3MFpjZOoLReUli25PAMjP7kJnlm9lvE3zIlEzxXJJhCvT5qR2omWK+eGli+wmNJ35x9zjQRDBqTVVL0u9Dk9wem+c2s8+Y2Z7EtEg3wUg4+cPljTie9Ptg8usCbe4+nHR7FfCjxNRIN8E3iRiwGPgOsAW4z8yazeyrZpafwussAw6f2JB4LxsJvrlMtIyk9z35cSnoB8on3FcO9E3SFoJvI0PAK8BPCL5FNCVq7CD4hvZHBH+3K4Cfn9guc48CfX56ChhhwlyomZUS7Fj8RdLdK5K25wDLgebEXWk7VWdivvxWgtFppbtXEIwsJ37lnwkT+9EIXOnuFUk/Re5+NLG/4QvuvoFgdH018JEUXqOZ4IMCADMzgvf26CRtj5H0vhPsq0jVbmCNmSWPyM9N3H+SxLeA6919ibufRZAJzyRtf8zdL3T3KoKpszOSt8vcokCfhxLTH18A/sHMrkh8na4nmAJpIhiFnnCBmV2bGM3/IcEHwdbEthZgTZrKWgBECXa65ZnZZzl5pDlb7gD+2sxWAZhZrZldk/j9HWZ2tgVr7XsJpmBSWab5PeC9ZnZ5YkT/GYL38skp2v6ZmVWa2XLg06kW7u77gB3A5xI7cN9PsM/gh5O1N7O1iR3VuWZ2JcE+hC8lbT8/8e+jnGCaqMndt6Raj8wuBfo8ldiJ+ecE/0l7gacJRqaXJ+aNT/gJwfxvF8EI7drEHDDA3wB/mZia+OM3WNIWgjn2fQRTDMOMn3aYTV8nWK73H2bWR/ABdnFi2xLgBwTv2R7gMWDaA57cfS/B6qF/IJjSeh/BDuXIJM2/QPAeHAT+g/EfsJjZHWZ2x2u83HXAJoK/2ZeBD7h7W+Kxb7Xx6/MvIFgF00fw97ze3ZNH83+aqLeRYDru/dP1VTLHdIELmYoFB5msc/cbMl2LiExPI3QRkSyhQBcRyRKachERyRIaoYuIZImMnYiopqbG6+vrM/XyIiKhtH379nZ3n/Rka9MGupndRXDwRKu7n3QOh8QBEl8HriI4Mu6j7v7cdM9bX1/Ps88+O10zERFJYmZTHjmcypTL3QSH/E7lSuC0xM/NwDdPpTgREUmPaQPd3R8HOl+jyTXAv3lgK1BhZkvTVaCIiKQmHTtF6xh/RF8Tk59wCDO72cyeNbNn29ra0vDSIiJyQjoCfbKTJ026FtLd73T3Te6+qbZ2qguoiIjI65GOQG9i/Jnhks/GJyIisyQdgf4A8BELbAZ63P1YGp5XRCTUYvEYD+17iC8+9kUe2vcQsfjMXj89lWWL9wKXEVwQoYng8lj5AO5+B8GltK4C9hMsW7xppooVEQmLWDzGe+55D08ffZqByAClBaVcXHcxW27YQm5O7oy85rSB7u4fmma7A7+XtopEREKmb3iUF4/2MjQaxTCGRmM0j/w3TzVuZTA6AEB/pJ+njz7NI/sf4erTr56ROjJ2pKiISJi4O73DUVp6hznQ2s+Btn72tvTz8rFeDnUMMBobvxakO+/HDOYNjls20h8Z4BcHnlagi4jMtMFIlFda+tl2qJO2vhEcePFoD7ubexmKxIjE4uPaL68sZv3Sci5fv5gLVlWyoCiIVHf4/u7jfHPXjxiODY61Ny/kYPPMrfBToItI1ovFna7BCJFonIGRKMd7h+kciNA1EKGxa4hthzrZe7yPkeirgV2Ql0OOwYKifK46eynlxXlUlxZQmJfL2toyzlmxkPKi/Clf86LVH2VX973j5tCr8jaw/8g6RqIxCvPSP4+uQBeRUIvFnfb+Edr6RjjcMUg0Hqeld5iuwVEaOwfpGRrlpeZeOgYmu9ofFOXnsKamjBs3r6KqrIAl5UW8ZW0NSxYWvaG6cnNy2XLDFh7Z/wg7ju/gvCXnkTe6kVu+8zw7G3u4aHXVG3r+ySjQRWTOcne6BkfpHoxwuGOQA239NLQP0NIzTEP7AG19IwxEokx2WYfcHKOuopiFxflsXlvNplWVlBTkUlyQR0VxPssS22rKCgjOMZh+uTm5XH361WNz5gfagsu5Hu0eBBToIpIF3J3+kSiHOwbpHhylsWuQgZEo0bgzGIlxpGOAI52DvNLST99IdNxjK0vyWbqwmLW1ZbzzzEWUFuRSW15EbVkByyqKKSnIo6Ikn+rSmQvq12vZwmIAmruHZ+T5FegiMmPcg4DefriLw52DbD3QQUP7AAfa+olE45M+xiwIvuWVxVxz/jLW1JSxsDifVdUlrKkto6q0YJZ7kT7FBblcsq6ahcVTz72/EQp0EXld4nFnIBKMsl882kNb3wgdAxGau4doaB+gZ2iUwZEoA5FXj45cXF7IhqXlXLqumuqyQpZXFlNdWsiKqmIWFOWTn2vk5+aQn5u9F1P77ic2z9hzK9BF5CSNnYPsbu6ld2iU473D9A6N0tQ1ROdghN6hUfqGoxzrGSI+Ye66rDCPJQuLWFtbSlVpIUX5OVSWFLB+aTlnLllAXUUxOTlzaxokmyjQReYhd+fl433sa+mjrW+Eg+0D9A1HOdo9xOGOAdr7x68IKcrPoa6imJqyQlZUlbCgKI8l5UUsKMpnWUURG5aWs6KqhKL8mTmkXVKjQBfJEu5OS+8InQMR2vtH6BqM0NI7zAtHe2npHaapc5DRuDMai9M3HCWWNLyuLMmnvDifJeVFvGv9Yk5bvICL6quoKMlncXkRBXnZOwWSTRToIiEzGouzq6mH5490cbhjkF1N3QxEYjR3DzEYOflsfiuqgnnqzWurKczLJT/XKC3Mo766hPNWVFJdVkBNWWEGeiLppkAXmWPcnfb+CAfa+jmY2Ll4rHuIZw930dg5SP9IdGzuujAvhwtWVbKsopi3nlbDmppSasoKqS4rpKq0gPKiPBaVv7EDZCQ8FOgiGdY5EGFHYxfPH+nmsX1tHGofoHd4/NrrovwcNq2qYuPKSipK8tmwtJxN9VUzelCMhI8CXWQWRGNxjvcO09I7zNHuYZ5u6KCld4Qdjd20948AkGNw9vIKrjmvjjW1paypLWNNTSlVpQUU5eeSq9UhMg0FukiaDEViY4emN7T109A2QHP3EC19wxzrHiaatBNyQWEei8oLefPaajYsLef8lRWcXbeQ0kL9l5TXT/96RE7R8GiMfS19PPFKO//1Sjv9ibP3tfWNjLUxg7qK4GjH81dU8uvnFrO8soQlC4uoLSvkjCULsvrgGckMBbrIFKKxOG39I+w93seeY33sOdbLS8d6aWjrH9sp+aa6cmrKgqMf6yqLWbeojDW1pdRXl2pNtsw6BbpIQmvfME8d6GBXUw/bDnXy4tGecUdC1lUUs37pAq560xLWLy3n3BUVLKsozlzBIhMo0GVeGh4N5ru3Hexk+5FudjZ2c6QzuLJMQV4OZ9ct5GOXrGZVTSnrasvYsLSchSUzc0IlkXRRoEvWcvfgqjSDo2w/3Mnu5l4OdwzS0N5PU9fQ2Dm0axcUsmlVJTduXsWFq6s4u26hVpRIKCnQJdR6h0dp7h7iWM8wHf0RWvuGef5IN50DEY73DHO0e2isbXlRHiuqgqMjrz1/OSurSrhodRXLK4u1lluyggJdQiEedzoHI7T1jfDckS5+saeVfS19NHUNndR2TW0pS8qLOG9lBTddUk91WQFnLC5n/dIFCm7Jagp0mXPcncMdgxxsH+Bg+wD72/p5bG/buNH2iqpiNq6s5EMXrWRVdQlLFxZTU1ZARXGB5rpl3lKgS0a4O/tb+9lzvI/GzkGOdAxyKHHZsba+kXEH4ZQV5rFxVSU3XVLPkoVFnL54AactKtNoW2QCBbrMmHjcae4Z4nAirA93DHKofYC2/hGO9wxzrOfV6yrWlBVSX13Cm9dWs6S8iJVVJSyvLGH90gVUlhTooggiKVCgS1oMRWIcbB/gxeYeGjsHefl4H88fefU8JRAsB1xVVcLi8iIuWl3FhfXBz/LKYh3yLpIG+l8kKXN3GtoH2N/az6H2AQ4lRtyHOgbGjbZzDNbUlnHJumouWl3F6prgyMkl5UUaaYvMIAW6TKpveJQdjd0c6w6W/h3vGebllj52NnaPtakqLRibJlldXUp9TSkblpWzsqpE5ykRyQAFugBwrGeIX77cSntfhK0NHTxzqHPsEmU5Fsxx1y4o5C/fu54L66uoryllYbFWk4jMJQr0eWp4NMaeY71sbehk++FOfvFy69iRk0sXFnHz29ZwydoaVlWXsKi8kMI8nWhKZK5ToM8TkWicvcf72NrQweOvtPH0wU4i0TgQHEF5y9vX8psbg6MndUFgkXBSoGehWNzZc6yX/a39tPYN84s9rTx/pJtILAjw0xeXcePmVZy3ooKNqyqp0xkDRbKCAj0LDEWCCy48d6SLV1r72Xqgg4b2gbHtyyuL+egl9ZyzfCGbVlWxZKEuGiySjRToIRSPO41dgzy+r41/f7aR3c29Y/PfFSX5nL54AR+7dDWb11RRu6BIOy9F5gkFekjE487zjd1s2X2c+587OnbAzrpFZXz6naexYWk5Zy0rZ0VVSYYrFZFMSSnQzewK4OtALvAtd//yhO0LgXuAlYnn/Jq7fzvNtc4bg5Eou5p6eKGph67BCJ0DEX6+p5X2/hHycox3nLmId5yxiPNXVrB+aXmmyxWROWLaQDezXOAbwLuBJmCbmT3g7i8lNfs94CV3f5+Z1QJ7zey77h6ZkaqzzImzCz6ws5nH9rWxs7F77ORUeTlGSUEum9dU895zlnLZGYs0hSIik0plhH4RsN/dGwDM7D7gGiA50B1YYMHp78qATiCa5lqzSmvvMP+5t5Xth7vY2tDJkc5BzODc5RX8ztvWsHFlJRtXVlBVWqCzCopISlIJ9DqgMel2E3DxhDb/CDwANAMLgN9y9/jEJzKzm4GbAVauXPl66g21oUiMn+9p4Zcvt/LgzmaicaeiJJ8LVlZyw+aVvPecZVpCKCKvWyqBPtnw0Cfcfg+wA3gnsBZ41MyecPfecQ9yvxO4E2DTpk0TnyMrjcbibG3o4KGdx/jpC8foH4lSUZLPb5xfx2+/uZ6zlpXrhFUikhapBHoTsCLp9nKCkXiym4Avu7sD+83sIHAm8Exaqgyh7sEID+46xj8/3sCRzkFKC3K56uyl/OYFy7mwvkoXIRaRtEsl0LcBp5nZauAocB3w4QltjgCXA0+Y2WLgDKAhnYWGQSQa5yc7jnLvM0d47kg3AGctK+f26zfyzjMXUZSv86GIyMyZNtDdPWpmnwK2ECxbvMvdd5vZLYntdwBfBO42sxcIpmhudff2Gax7TmnqGuQrP9vLf77cSv9IlLqKYv7Xu07nbafXcN6KCu3UFJFZkdI6dHd/GHh4wn13JP3eDPxaekub29ydpw928v1nm/jR803k5eRw7cY6fu2sxbzjjEUKcRGZdTpS9BRFonEefamFu588yLZDXZQU5PKRN9fz8UtX6yhNEckoBXqKItE4//bUIf75iQZaekeoKSvki9ecxf/YtEJz4yIyJyjQU9A1EOGT39nOM4c6ecvaav7m2rO5ZF2NLvogInOKAn0aDW39fOzubTT3DPP1687jmvPqMl2SiMikFOiv4dGXWvjM93aQl5vDvb9zMResqsp0SSIiU1KgT2I0FudrW/byT483cHbdQm6/fqN2eIrInKdAT+LuPHekmy8/sodth7q4YfNK/vK9G7TTU0RCQYGe4O781U9e5J6tR1hQlKf5chEJHQU6wQUl/urHu/nhc018/NLV/NG7T6e0UG+NiITLvE8td+cP7tvBz/e08AeXn8Yfvus0HeUpIqE0rwM9Gotz6w9f4NGXWvizK8/kk29fm+mSRERet3kb6JFonI//6zaeeKWd/3nZWm5+25pMlyQi8obMy0B3d265ZztPvNLOZ6/ewMcuXZ3pkkRE3rCcTBeQCf/0eAO/fLmV2648U2EuIllj3gX6t55o4MuPvMyvbVjMzW/VNIuIZI95Feg/2N7El366h3dvWMw3b7hA1/IUkawybwL9aPcQf3b/Lt68pprbr9+oa3qKSNaZF4Hu7nz2xy+SY8bXPngu+bnzotsiMs/Mi2T70fNH+cXLrfzJe86grqI40+WIiMyIrA/0gZEof/3TPVywqpKbLtGKFhHJXlkf6P/yXwfpGIjwF+9dr3lzEclqWR3orX3DfPNXB7jyTUvYuLIy0+WIiMyorA70f3qsgUgszp9ecWamSxERmXFZG+itvcPcs/Uwv3FeHatrSjNdjojIjMvaQP/mYweIxp3fv3xdpksREZkVWRnobX0jfPfpI1x7fh2rqjU6F5H5ISsD/cGdzUSicT75dp2rRUTmj6wM9Ad2NrNhaTnrFi3IdCkiIrMm6wL9cMcAOxq7+fXzlmW6FBGRWZV1gf7QrmMAXH3O0gxXIiIyu7Iu0B9+4Rjnr6xgeWVJpksREZlVWRXoe471sru5l6vP0XSLiMw/WRXoW3Yfxwzef35dpksREZl1WRXoT7zSzjl1C6kqLch0KSIisy5rAn0oEmNnYzeXrKvJdCkiIhmRNYG+s6mbaNzZVK+zKorI/JRSoJvZFWa218z2m9ltU7S5zMx2mNluM3ssvWVOb/vhLgDOX6FAF5H5KW+6BmaWC3wDeDfQBGwzswfc/aWkNhXA7cAV7n7EzBbNUL1Teu5wF2trS6nU/LmIzFOpjNAvAva7e4O7R4D7gGsmtPkwcL+7HwFw99b0ljm9nU09nKfRuYjMY6kEeh3QmHS7KXFfstOBSjP7lZltN7OPpKvAVLT1jdDeP8KGZeWz+bIiInPKtFMuwGQX4vRJnucC4HKgGHjKzLa6+75xT2R2M3AzwMqVK0+92insOdYLwPqlOhmXiMxfqYzQm4AVSbeXA82TtPmZuw+4ezvwOHDuxCdy9zvdfZO7b6qtrX29NZ9k7/E+AM5cohG6iMxfqQT6NuA0M1ttZgXAdcADE9r8BHirmeWZWQlwMbAnvaVOraG9n6rSAh1QJCLz2rRTLu4eNbNPAVuAXOAud99tZrcktt/h7nvM7GfALiAOfMvdX5zJwpMdbB+gvlon4xKR+S2VOXTc/WHg4Qn33THh9t8Cf5u+0lJ3sH2AS9elbwpHRCSMQn+k6MBIlJbeEVbXaIQuIvNb6AP9UMcAAKtryjJciYhIZoU+0A+2B4FerxG6iMxzoQ/0QycCvbo0w5WIiGRW6AO9oX2AxeWFlBamtH9XRCRrhT7Qj3UP6/qhIiJkQaC39A2zpLwo02WIiGRc6AO9tXeEReWFmS5DRCTjQh3o/SNR+keiLNYIXUQk3IHe2jsMwGKN0EVEwh3ox8cCXSN0EZFQB3pr7wigQBcRgZAHeotG6CIiY0Ie6COUFORSpoOKRETCHejdgxFd1EJEJCHUgd47HKW8KD/TZYiIzAkhD/RRyos13SIiAmEP9KFRFmiELiIChDzQ+zTlIiIyJtSB3jukKRcRkRNCG+jxuNMfiWrKRUQkIbSB3jcSxR3KizRCFxGBEAd679AoAOXFGqGLiECYA304EegaoYuIACEO9MFIDICSAgW6iAiEONCHxgI9N8OViIjMDeEN9NEg0IvyFegiIhDmQNcIXURknPAGemKEXqxAFxEBwhzoiRF6saZcRESAMAe6RugiIuOEN9AjMXIMCnJD2wURkbQKbRoOjcYozs/FzDJdiojInBDaQB+MxCjWQUUiImNCG+jDozGKC0JbvohI2oU2EYciMUryNUIXETkhtIE+OBqjSCtcRETGpBToZnaFme01s/1mdttrtLvQzGJm9oH0lTi54UiM4vzQfh6JiKTdtIloZrnAN4ArgQ3Ah8xswxTtvgJsSXeRkzmxykVERAKpDHEvAva7e4O7R4D7gGsmafdp4IdAaxrrm1IkGqcgTyN0EZETUknEOqAx6XZT4r4xZlYHvB+447WeyMxuNrNnzezZtra2U611nNF4nDwdVCQiMiaVRJzsyB2fcPvvgFvdPfZaT+Tud7r7JnffVFtbm2KJk4vGnPwcHVQkInJCKuv+moAVSbeXA80T2mwC7ksctVkDXGVmUXf/cTqKnEws7uTmaIQuInJCKoG+DTjNzFYDR4HrgA8nN3D31Sd+N7O7gYdmMswBRmNx8nM1QhcROWHaQHf3qJl9imD1Si5wl7vvNrNbEttfc958pkTjTp4CXURkTEqHWrr7w8DDE+6bNMjd/aNvvKzpjcbi5GnKRURkTGgTMRZ38rRTVERkTGgDPRpzLVsUEUkS2kQcjWunqIhIslAGeizuuKM5dBGRJKFMxGg8DqBVLiIiScIZ6LHgQFXtFBUReVW4A107RUVExoQyEUcTUy7aKSoi8qpQBnosHozQczXlIiIyJpSBPhpLjNC1ykVEZEwoE/HVOXSN0EVETghnoI8tWwxl+SIiMyKUiRiNa9miiMhE4Qx0rUMXETlJKAN9bKeoplxERMaEMhHHply0U1REZEw4Az2mdegiIhOFM9DjmnIREZkolImonaIiIicLZaBrp6iIyMlCmYg6l4uIyMlCGeijiUDX2RZFRF4VykCPJqZcdAk6EZFXhTIRdXIuEZGThTPQx87lEsryRURmRCgTUReJFhE5WSgDfTQx5aILXIiIvCqUiTi2U1QjdBGRMeEMdK1DFxE5STgD/cSUi44UFREZE8pEjMbjmGmELiKSLJSBPhpz7RAVEZkglKkYi8c1OhcRmSCUgT4ac61wERGZIJSBHo3HtUNURGSCUKZiNOa6uIWIyAQpBbqZXWFme81sv5ndNsn2681sV+LnSTM7N/2lvioaV6CLiEw0baCbWS7wDeBKYAPwITPbMKHZQeDt7n4O8EXgznQXmiwai5OnKRcRkXFSScWLgP3u3uDuEeA+4JrkBu7+pLt3JW5uBZant8zxRuPaKSoiMlEqgV4HNCbdbkrcN5WPA49MtsHMbjazZ83s2ba2ttSrnCAai2sduojIBKmk4mRDYZ+0odk7CAL91sm2u/ud7r7J3TfV1tamXuUEsbhrHbqIyAR5KbRpAlYk3V4ONE9sZGbnAN8CrnT3jvSUN7nRmOt6oiIiE6QyQt8GnGZmq82sALgOeCC5gZmtBO4HbnT3fekvc7xoXDtFRUQmmnaE7u5RM/sUsAXIBe5y991mdkti+x3AZ4Fq4HYzA4i6+6aZKnpU69BFRE6SypQL7v4w8PCE++5I+v0TwCfSW9rUorE4xQW5s/VyIiKhEMp5i1jcdYFoEZEJQpmK2ikqInKyUAZ6NB7XCF1EZIJQpmI05uRqhC4iMk44Az3u5GuVi4jIOOEMdJ2cS0TkJKFMxdG4doqKiEwUykCPxnRNURGRicIZ6FqHLiJyklCmYlTr0EVEThLOQNfJuUREThK6VHR3nZxLRGQSoQv0eOLSGppDFxEZL3SpGInGAcjP0whdRCRZ6AK9Y2AEgOrSggxXIiIyt4Qu0Nv7IwDUlBVmuBIRkbklfIHeF4zQFegiIuOFLtArSvK54qwlLK0oynQpIiJzSkqXoJtLNtVXsam+KtNliIjMOaEboYuIyOQU6CIiWUKBLiKSJRToIiJZQoEuIpIlFOgiIllCgS4ikiUU6CIiWcLcPTMvbNYGHH6dD68B2tNYThioz/OD+jw/vJE+r3L32sk2ZCzQ3wgze9bdN2W6jtmkPs8P6vP8MFN91pSLiEiWUKCLiGSJsAb6nZkuIAPU5/lBfZ4fZqTPoZxDFxGRk4V1hC4iIhMo0EVEssScDnQzu8LM9prZfjO7bZLtZmZ/n9i+y8w2ZqLOdEqhz9cn+rrLzJ40s3MzUWc6TdfnpHYXmlnMzD4wm/XNhFT6bGaXmdkOM9ttZo/Ndo3plsK/7YVm9qCZ7Uz0+aZM1JkuZnaXmbWa2YtTbE9/frn7nPwBcoEDwBqgANgJbJjQ5irgEcCAzcDTma57Fvr8FqAy8fuV86HPSe1+CTwMfCDTdc/C37kCeAlYmbi9KNN1z0Kf/xz4SuL3WqATKMh07W+gz28DNgIvTrE97fk1l0foFwH73b3B3SPAfcA1E9pcA/ybB7YCFWa2dLYLTaNp++zuT7p7V+LmVmD5LNeYbqn8nQE+DfwQaJ3N4mZIKn3+MHC/ux8BcPew9zuVPjuwwMwMKCMI9Ojslpk+7v44QR+mkvb8msuBXgc0Jt1uStx3qm3C5FT783GCT/gwm7bPZlYHvB+4Yxbrmkmp/J1PByrN7Fdmtt3MPjJr1c2MVPr8j8B6oBl4AfgDd4/PTnkZkfb8mssXibZJ7pu4xjKVNmGScn/M7B0EgX7pjFY081Lp898Bt7p7LBi8hV4qfc4DLgAuB4qBp8xsq7vvm+niZkgqfX4PsAN4J7AWeNTMnnD33hmuLVPSnl9zOdCbgBVJt5cTfHKfapswSak/ZnYO8C3gSnfvmKXaZkoqfd4E3JcI8xrgKjOLuvuPZ6XC9Ev133a7uw8AA2b2OHAuENZAT6XPNwFf9mCCeb+ZHQTOBJ6ZnRJnXdrzay5PuWwDTjOz1WZWAFwHPDChzQPARxJ7izcDPe5+bLYLTaNp+2xmK4H7gRtDPFpLNm2f3X21u9e7ez3wA+B3QxzmkNq/7Z8AbzWzPDMrAS4G9sxynemUSp+PEHwjwcwWA2cADbNa5exKe37N2RG6u0fN7FPAFoI95He5+24zuyWx/Q6CFQ9XAfuBQYJP+NBKsc+fBaqB2xMj1qiH+Ex1KfY5q6TSZ3ffY2Y/A3YBceBb7j7p8rcwSPHv/EXgbjN7gWA64lZ3D+1pdc3sXuAyoMbMmoDPAfkwc/mlQ/9FRLLEXJ5yERGRU6BAFxHJEgp0EZEsoUAXEckSCnQRkSyhQBcRyRIKdBGRLPH/Act6Uw571nF1AAAAAElFTkSuQmCC\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "f1_thresholds = []\n",
    "for model_name, inferences in model_inferences.items():\n",
    "    labels = inferences['labels']\n",
    "    predictions = inferences['predictions']\n",
    "    thresholds = np.arange(0.0, 1.0, 0.001)\n",
    "    f1scores = np.zeros(shape=(len(thresholds)))\n",
    "    # Sweep across the thresholds and calculate the f1-score for those predictions.\n",
    "    # The idea here is that the threshold with the highest f1-score is ideal\n",
    "    for index, elem in tqdm(enumerate(thresholds), total=len(thresholds), desc=\"Calculating F-Scores\"):\n",
    "        # Corrected probabilities\n",
    "        y_pred_prob = (np.array(predictions) > elem).astype('int')\n",
    "        # Calculate the f-score\n",
    "        f1scores[index] = f1_score(np.array(labels), y_pred_prob)\n",
    "\n",
    "    # Find the optimal threshold\n",
    "    f1score_argmax_index = np.argmax(f1scores)\n",
    "    optimal_f1score_threshold = thresholds[f1score_argmax_index]\n",
    "    f1score_at_optimal_threshold = f1scores[f1score_argmax_index]\n",
    "    plt.plot(thresholds, f1scores)\n",
    "    plt.plot(optimal_f1score_threshold, f1score_at_optimal_threshold, color=\"g\", marker=\"o\", markersize=5)\n",
    "    print(f'Optimal Threshold from F1-Score Thresholding {optimal_f1score_threshold:.2f}')\n",
    "    f1_thresholds.append(optimal_f1score_threshold)\n",
    "plt.title(f\"Optimal Threshold: {np.array(f1_thresholds).mean():.2f}\")\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ]
}